2024-09-21T09:33:46.154 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T09:33:46.180 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:33:46.268 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913660 branch: main description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - echo wait for mgr daemons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo wait for minority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for majority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for all mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for half of osds to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done first-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7913660' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: main sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - sleep 60 - echo wait for upgrade to complete - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 5357 sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 targets: smithi032.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNVZEfRpyrp+y270IbvUmrNZzgm121cG0eTLoikKcgAPbBcSYVPENc+9nuIQwNmN7KoHlRlL6d1cC9ElfabrS8w= smithi045.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM3b5uiHwe60YPOb+M+N81pbOrjHYeOKotnlDQhB63fqgsZKxKAt3moFe6Bm5lU073F7ghOgG4IxLF02zZAJNss= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install nvmetcli nvme-cli -y - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_03:08:26 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 2024-09-21T09:33:46.268 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa; will attempt to use it 2024-09-21T09:33:46.269 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks 2024-09-21T09:33:46.269 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T09:33:46.271 INFO:teuthology.task.internal:Checking packages... 2024-09-21T09:33:46.291 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '62da54b4b83a96ca9976b7e1d0c8272a564a2208' 2024-09-21T09:33:46.292 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:33:46.292 INFO:teuthology.packaging:ref: None 2024-09-21T09:33:46.292 INFO:teuthology.packaging:tag: None 2024-09-21T09:33:46.292 INFO:teuthology.packaging:branch: main 2024-09-21T09:33:46.292 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:33:46.292 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-21T09:33:46.499 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5092.g62da54b4 2024-09-21T09:33:46.501 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T09:33:46.508 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T09:33:46.508 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T09:33:46.514 INFO:teuthology.task.internal:Saving configuration 2024-09-21T09:33:46.538 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T09:33:46.545 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T09:33:46.569 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi032.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913660', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:29:14.110416', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNVZEfRpyrp+y270IbvUmrNZzgm121cG0eTLoikKcgAPbBcSYVPENc+9nuIQwNmN7KoHlRlL6d1cC9ElfabrS8w='} 2024-09-21T09:33:46.591 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi045.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913660', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:29:14.112317', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM3b5uiHwe60YPOb+M+N81pbOrjHYeOKotnlDQhB63fqgsZKxKAt3moFe6Bm5lU073F7ghOgG4IxLF02zZAJNss='} 2024-09-21T09:33:46.591 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T09:33:46.597 INFO:teuthology.task.internal:roles: ubuntu@smithi032.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-21T09:33:46.597 INFO:teuthology.task.internal:roles: ubuntu@smithi045.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-21T09:33:46.597 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T09:33:46.655 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fc3c3aa4160>, signals=[15]) 2024-09-21T09:33:46.655 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T09:33:46.662 INFO:teuthology.task.internal:Opening connections... 2024-09-21T09:33:46.662 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi032.front.sepia.ceph.com 2024-09-21T09:33:46.664 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:33:46.749 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi045.front.sepia.ceph.com 2024-09-21T09:33:46.749 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:33:46.821 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T09:33:46.833 DEBUG:teuthology.orchestra.run.smithi032:> uname -m 2024-09-21T09:33:46.851 INFO:teuthology.orchestra.run.smithi032.stdout:x86_64 2024-09-21T09:33:46.852 DEBUG:teuthology.orchestra.run.smithi032:> cat /etc/os-release 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:NAME="CentOS Stream" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:VERSION="9" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:ID="centos" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:VERSION_ID="9" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:ANSI_COLOR="0;31" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:33:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:33:46.910 INFO:teuthology.orchestra.run.smithi032.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:33:46.910 INFO:teuthology.lock.ops:Updating smithi032.front.sepia.ceph.com on lock server 2024-09-21T09:33:46.928 DEBUG:teuthology.orchestra.run.smithi045:> uname -m 2024-09-21T09:33:46.946 INFO:teuthology.orchestra.run.smithi045.stdout:x86_64 2024-09-21T09:33:46.946 DEBUG:teuthology.orchestra.run.smithi045:> cat /etc/os-release 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:NAME="CentOS Stream" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:VERSION="9" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:ID="centos" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:VERSION_ID="9" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:ANSI_COLOR="0;31" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:33:47.004 INFO:teuthology.orchestra.run.smithi045.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:33:47.005 INFO:teuthology.lock.ops:Updating smithi045.front.sepia.ceph.com on lock server 2024-09-21T09:33:47.028 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T09:33:47.037 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T09:33:47.059 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T09:33:47.059 DEBUG:teuthology.orchestra.run.smithi032:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:33:47.061 DEBUG:teuthology.orchestra.run.smithi045:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:33:47.078 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T09:33:47.084 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T09:33:47.084 DEBUG:teuthology.orchestra.run.smithi032:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:33:47.119 DEBUG:teuthology.orchestra.run.smithi045:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:33:47.147 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T09:33:47.200 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T09:33:47.214 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T09:33:47.214 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T09:33:47.214 DEBUG:teuthology.orchestra.run.smithi032:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:33:47.215 DEBUG:teuthology.orchestra.run.smithi045:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:33:47.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:33:47.231 DEBUG:teuthology.orchestra.run.smithi045:> uname -r 2024-09-21T09:33:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:33:47.232 DEBUG:teuthology.orchestra.run.smithi032:> uname -r 2024-09-21T09:33:47.288 INFO:teuthology.orchestra.run.smithi045.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:33:47.288 INFO:teuthology.task.kernel:Running kernel on smithi045: 5.14.0-503.el9.x86_64 2024-09-21T09:33:47.288 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum install -y kernel 2024-09-21T09:33:47.290 INFO:teuthology.orchestra.run.smithi032.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:33:47.290 INFO:teuthology.task.kernel:Running kernel on smithi032: 5.14.0-503.el9.x86_64 2024-09-21T09:33:47.290 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum install -y kernel 2024-09-21T09:33:49.532 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.3 MB 00:00 2024-09-21T09:33:50.741 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - BaseOS 4.7 MB/s | 8.3 MB 00:01 2024-09-21T09:33:51.342 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - AppStream 30 MB/s | 20 MB 00:00 2024-09-21T09:33:54.050 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - AppStream 9.3 MB/s | 20 MB 00:02 2024-09-21T09:33:57.767 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - CRB 2.6 MB/s | 6.5 MB 00:02 2024-09-21T09:33:58.767 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - CRB 8.0 MB/s | 6.5 MB 00:00 2024-09-21T09:33:59.743 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - Extras packages 28 kB/s | 19 kB 00:00 2024-09-21T09:34:00.806 INFO:teuthology.orchestra.run.smithi032.stdout:Extra Packages for Enterprise Linux 24 MB/s | 23 MB 00:00 2024-09-21T09:34:01.126 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - Extras packages 17 kB/s | 19 kB 00:01 2024-09-21T09:34:01.897 INFO:teuthology.orchestra.run.smithi045.stdout:Extra Packages for Enterprise Linux 36 MB/s | 23 MB 00:00 2024-09-21T09:34:06.487 INFO:teuthology.orchestra.run.smithi032.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-21T09:34:07.510 INFO:teuthology.orchestra.run.smithi045.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-21T09:34:08.468 INFO:teuthology.orchestra.run.smithi032.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:34:08.468 INFO:teuthology.orchestra.run.smithi032.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:34:08.564 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:34:08.580 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:34:08.581 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:34:08.581 INFO:teuthology.orchestra.run.smithi032.stdout:Install 4 Packages 2024-09-21T09:34:08.581 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:34:08.581 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 79 M 2024-09-21T09:34:08.581 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 126 M 2024-09-21T09:34:08.581 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:34:08.680 INFO:teuthology.orchestra.run.smithi032.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 464 kB/s | 36 kB 00:00 2024-09-21T09:34:09.613 INFO:teuthology.orchestra.run.smithi032.stdout:(2/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 34 MB/s | 35 MB 00:01 2024-09-21T09:34:09.617 INFO:teuthology.orchestra.run.smithi045.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:34:09.617 INFO:teuthology.orchestra.run.smithi045.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:34:09.707 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:34:09.711 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout: Package Architecture Version Repository Size 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout:Install 4 Packages 2024-09-21T09:34:09.712 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:34:09.713 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 79 M 2024-09-21T09:34:09.713 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 126 M 2024-09-21T09:34:09.713 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:34:09.780 INFO:teuthology.orchestra.run.smithi032.stdout:(3/4): kernel-core-5.14.0-508.el9.x86_64.rpm 13 MB/s | 16 MB 00:01 2024-09-21T09:34:10.078 INFO:teuthology.orchestra.run.smithi045.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 472 kB/s | 36 kB 00:00 2024-09-21T09:34:10.131 INFO:teuthology.orchestra.run.smithi032.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 20 MB/s | 29 MB 00:01 2024-09-21T09:34:10.131 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:34:10.131 INFO:teuthology.orchestra.run.smithi032.stdout:Total 51 MB/s | 79 MB 00:01 2024-09-21T09:34:10.512 INFO:teuthology.orchestra.run.smithi045.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 31 MB/s | 16 MB 00:00 2024-09-21T09:34:10.842 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:34:11.129 INFO:teuthology.orchestra.run.smithi045.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 27 MB/s | 29 MB 00:01 2024-09-21T09:34:11.244 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:34:11.244 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:34:11.520 INFO:teuthology.orchestra.run.smithi045.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 23 MB/s | 35 MB 00:01 2024-09-21T09:34:11.521 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:34:11.521 INFO:teuthology.orchestra.run.smithi045.stdout:Total 44 MB/s | 79 MB 00:01 2024-09-21T09:34:12.207 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:34:12.208 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:34:12.234 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:34:12.619 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:34:12.619 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:34:13.605 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:34:13.606 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:34:14.774 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:34:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:34:15.549 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:34:16.171 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:34:16.843 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:34:16.939 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:34:18.307 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:34:18.578 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:34:19.687 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:34:19.816 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:34:23.713 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:34:23.938 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:24.939 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:34:24.984 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:29.081 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:30.123 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:35:08.279 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:35:08.321 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:35:08.623 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:35:08.623 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:35:08.623 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:35:08.623 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:35:09.361 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:35:09.402 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:35:09.475 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:35:09.475 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:35:09.475 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:35:09.475 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:35:09.475 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:35:09.475 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:35:09.475 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:35:09.475 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:35:09.475 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:35:09.728 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:35:09.728 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:35:09.728 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:35:09.728 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:35:09.792 DEBUG:teuthology.orchestra.run.smithi032:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:35:09.879 INFO:teuthology.orchestra.run.smithi032.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:35:09.879 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:35:09.879 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:35:09.879 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:35:09.879 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:35:09.879 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-21T09:35:09.879 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi032.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:35:09.879 DEBUG:teuthology.orchestra.run.smithi032:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:35:09.937 INFO:teuthology.orchestra.run.smithi032.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:35:09.938 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:35:09.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo rpm -qi grub2-tools 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Name : grub2-tools 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Epoch : 1 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Version : 2.06 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Release : 82.el9 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Architecture: x86_64 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Group : Unspecified 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Size : 8274670 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:License : GPLv3+ 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:35:09.969 INFO:teuthology.orchestra.run.smithi032.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:Packager : builder@centos.org 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:Vendor : CentOS 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:Summary : Support tools for GRUB. 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:Description : 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:hardware devices. 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:35:09.970 INFO:teuthology.orchestra.run.smithi032.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:35:09.971 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:35:09.971 INFO:teuthology.task.kernel:Updating grub on smithi032 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:35:09.971 DEBUG:teuthology.orchestra.run.smithi032:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:35:10.592 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:35:10.592 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:35:10.593 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:35:10.593 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:35:10.593 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:35:10.593 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:35:10.593 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:35:10.593 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:35:10.593 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:35:10.795 INFO:teuthology.orchestra.run.smithi032.stderr:Generating grub configuration file ... 2024-09-21T09:35:10.940 DEBUG:teuthology.orchestra.run.smithi045:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:35:11.022 INFO:teuthology.orchestra.run.smithi045.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:35:11.022 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:35:11.022 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:35:11.022 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:35:11.022 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:35:11.023 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-21T09:35:11.023 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi045.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:35:11.023 DEBUG:teuthology.orchestra.run.smithi045:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:35:11.081 INFO:teuthology.orchestra.run.smithi045.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:35:11.082 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:35:11.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo rpm -qi grub2-tools 2024-09-21T09:35:11.123 INFO:teuthology.orchestra.run.smithi045.stdout:Name : grub2-tools 2024-09-21T09:35:11.123 INFO:teuthology.orchestra.run.smithi045.stdout:Epoch : 1 2024-09-21T09:35:11.123 INFO:teuthology.orchestra.run.smithi045.stdout:Version : 2.06 2024-09-21T09:35:11.123 INFO:teuthology.orchestra.run.smithi045.stdout:Release : 82.el9 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Architecture: x86_64 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Group : Unspecified 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Size : 8274670 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:License : GPLv3+ 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Packager : builder@centos.org 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Vendor : CentOS 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Summary : Support tools for GRUB. 2024-09-21T09:35:11.124 INFO:teuthology.orchestra.run.smithi045.stdout:Description : 2024-09-21T09:35:11.125 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:35:11.125 INFO:teuthology.orchestra.run.smithi045.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:35:11.125 INFO:teuthology.orchestra.run.smithi045.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:35:11.125 INFO:teuthology.orchestra.run.smithi045.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:35:11.125 INFO:teuthology.orchestra.run.smithi045.stdout:hardware devices. 2024-09-21T09:35:11.125 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:35:11.125 INFO:teuthology.orchestra.run.smithi045.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:35:11.126 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:35:11.126 INFO:teuthology.task.kernel:Updating grub on smithi045 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:35:11.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:35:11.829 INFO:teuthology.orchestra.run.smithi045.stderr:Generating grub configuration file ... 2024-09-21T09:35:12.297 INFO:teuthology.orchestra.run.smithi032.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:35:12.333 INFO:teuthology.orchestra.run.smithi032.stderr:done 2024-09-21T09:35:12.335 DEBUG:teuthology.orchestra.run.smithi032:> mktemp 2024-09-21T09:35:12.350 INFO:teuthology.orchestra.run.smithi032.stdout:/tmp/tmp.5GnXWF7SDe 2024-09-21T09:35:12.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.5GnXWF7SDe 2024-09-21T09:35:12.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 0666 /tmp/tmp.5GnXWF7SDe 2024-09-21T09:35:12.554 DEBUG:teuthology.orchestra.remote:smithi032:/tmp/tmp.5GnXWF7SDe is 6KB 2024-09-21T09:35:12.602 DEBUG:teuthology.orchestra.run.smithi032:> rm -fr /tmp/tmp.5GnXWF7SDe 2024-09-21T09:35:12.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:35:12.678 INFO:teuthology.orchestra.run.smithi032.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:35:12.678 INFO:teuthology.orchestra.run.smithi032.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:35:12.679 INFO:teuthology.orchestra.run.smithi032.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:35:12.679 INFO:teuthology.orchestra.run.smithi032.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:35:12.680 DEBUG:teuthology.orchestra.run.smithi032:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:35:12.955 DEBUG:teuthology.orchestra.run.smithi032:> sudo shutdown -r now 2024-09-21T09:35:13.197 INFO:teuthology.orchestra.run.smithi045.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:35:13.228 INFO:teuthology.orchestra.run.smithi045.stderr:done 2024-09-21T09:35:13.230 DEBUG:teuthology.orchestra.run.smithi045:> mktemp 2024-09-21T09:35:13.246 INFO:teuthology.orchestra.run.smithi045.stdout:/tmp/tmp.DeDXIEcPTK 2024-09-21T09:35:13.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.DeDXIEcPTK 2024-09-21T09:35:13.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod 0666 /tmp/tmp.DeDXIEcPTK 2024-09-21T09:35:13.443 DEBUG:teuthology.orchestra.remote:smithi045:/tmp/tmp.DeDXIEcPTK is 6KB 2024-09-21T09:35:13.492 DEBUG:teuthology.orchestra.run.smithi045:> rm -fr /tmp/tmp.DeDXIEcPTK 2024-09-21T09:35:13.506 DEBUG:teuthology.orchestra.run.smithi045:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:35:13.568 INFO:teuthology.orchestra.run.smithi045.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:35:13.569 INFO:teuthology.orchestra.run.smithi045.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:35:13.569 INFO:teuthology.orchestra.run.smithi045.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:35:13.569 INFO:teuthology.orchestra.run.smithi045.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:35:13.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:35:13.727 DEBUG:teuthology.orchestra.run.smithi045:> sudo shutdown -r now 2024-09-21T09:35:42.986 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-21T09:35:42.987 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-09-21T09:35:42.987 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:35:43.730 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-21T09:35:43.730 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-21T09:35:43.730 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:36:43.049 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T09:36:43.733 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T09:36:52.057 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-09-21T09:36:52.057 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:36:52.429 DEBUG:teuthology.orchestra.run.smithi032:> true 2024-09-21T09:36:52.734 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-21T09:36:52.734 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:36:52.776 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-09-21T09:36:52.776 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:36:52.776 DEBUG:teuthology.orchestra.run.smithi032:> uname -r 2024-09-21T09:36:52.791 INFO:teuthology.orchestra.run.smithi032.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:36:52.791 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:36:52.791 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:36:52.791 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:36:53.070 DEBUG:teuthology.orchestra.run.smithi045:> true 2024-09-21T09:36:53.440 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-21T09:36:53.440 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:36:53.440 DEBUG:teuthology.orchestra.run.smithi045:> uname -r 2024-09-21T09:36:53.456 INFO:teuthology.orchestra.run.smithi045.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:36:53.456 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:36:53.456 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:36:53.456 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:36:53.792 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-21T09:36:53.792 DEBUG:teuthology.orchestra.run.smithi032:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:36:53.857 INFO:teuthology.orchestra.run.smithi032.stdout:ttyS1 2024-09-21T09:36:53.875 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:54.457 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-21T09:36:54.457 DEBUG:teuthology.orchestra.run.smithi045:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:36:54.551 INFO:teuthology.orchestra.run.smithi045.stdout:ttyS1 2024-09-21T09:36:54.568 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:54.568 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T09:36:54.575 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T09:36:54.576 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:36:54.578 DEBUG:teuthology.orchestra.run.smithi045:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:36:54.595 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T09:36:54.606 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T09:36:54.638 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T09:36:54.639 DEBUG:teuthology.orchestra.run.smithi032:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:36:54.641 DEBUG:teuthology.orchestra.run.smithi045:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:36:54.673 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T09:36:54.680 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T09:36:54.680 DEBUG:teuthology.orchestra.run.smithi032:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:36:54.715 DEBUG:teuthology.orchestra.run.smithi045:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:36:54.751 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:36:54.753 INFO:teuthology.orchestra.run.smithi045.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:36:54.763 INFO:teuthology.orchestra.run.smithi045.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:36:54.763 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:36:54.765 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T09:36:54.772 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T09:36:54.772 DEBUG:teuthology.orchestra.run.smithi032:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:36:54.811 DEBUG:teuthology.orchestra.run.smithi045:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:36:54.837 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T09:36:54.846 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T09:36:54.846 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:36:54.883 DEBUG:teuthology.orchestra.run.smithi045:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:36:54.898 DEBUG:teuthology.orchestra.run.smithi032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:36:54.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:36:55.039 DEBUG:teuthology.orchestra.run.smithi032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:36:55.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:36:55.171 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:36:55.171 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:36:55.237 DEBUG:teuthology.orchestra.run.smithi045:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:36:55.279 DEBUG:teuthology.orchestra.run.smithi045:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:36:55.324 DEBUG:teuthology.orchestra.run.smithi045:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:36:55.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:36:55.443 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:36:55.443 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:36:55.511 DEBUG:teuthology.orchestra.run.smithi032:> sudo service rsyslog restart 2024-09-21T09:36:55.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo service rsyslog restart 2024-09-21T09:36:55.554 INFO:teuthology.orchestra.run.smithi032.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:36:55.597 INFO:teuthology.orchestra.run.smithi045.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:36:56.067 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T09:36:56.085 INFO:teuthology.task.internal:Starting timer... 2024-09-21T09:36:56.085 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T09:36:56.131 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T09:36:56.150 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-09-21T09:36:56.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo service auditd rotate 2024-09-21T09:36:56.204 INFO:teuthology.orchestra.run.smithi032.stdout:Rotating logs: 2024-09-21T09:36:56.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo service auditd rotate 2024-09-21T09:36:56.291 INFO:teuthology.orchestra.run.smithi045.stdout:Rotating logs: 2024-09-21T09:36:56.293 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T09:36:56.293 DEBUG:teuthology.orchestra.run.smithi032:> /usr/sbin/getenforce 2024-09-21T09:36:56.323 INFO:teuthology.orchestra.run.smithi032.stdout:Permissive 2024-09-21T09:36:56.323 DEBUG:teuthology.orchestra.run.smithi045:> /usr/sbin/getenforce 2024-09-21T09:36:56.350 INFO:teuthology.orchestra.run.smithi045.stdout:Permissive 2024-09-21T09:36:56.350 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi032.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi045.front.sepia.ceph.com': 'permissive'} 2024-09-21T09:36:56.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-21T09:36:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:36:56.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-21T09:36:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:36:56.418 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T09:36:56.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:36:56.467 DEBUG:teuthology.orchestra.run.smithi045:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:36:56.497 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T09:36:56.507 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-21T09:36:56.509 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-21T09:36:56.607 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T09:36:56.617 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T09:36:56.618 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi032.front.sepia.ceph.com,smithi045.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T09:40:42.281 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi032.front.sepia.ceph.com'), Remote(name='ubuntu@smithi045.front.sepia.ceph.com')] 2024-09-21T09:40:42.282 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-09-21T09:40:42.283 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:40:42.358 DEBUG:teuthology.orchestra.run.smithi032:> true 2024-09-21T09:40:42.435 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-09-21T09:40:42.435 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-21T09:40:42.435 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:40:42.508 DEBUG:teuthology.orchestra.run.smithi045:> true 2024-09-21T09:40:42.591 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-21T09:40:42.592 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T09:40:42.600 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T09:40:42.600 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:40:42.600 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:40:42.602 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:40:42.603 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:40:42.632 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:40:42.648 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:40:42.665 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:40:42.674 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: ntpd: command not found 2024-09-21T09:40:42.681 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:40:42.686 INFO:teuthology.orchestra.run.smithi032.stdout:506 Cannot talk to daemon 2024-09-21T09:40:42.701 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:40:42.708 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: ntpd: command not found 2024-09-21T09:40:42.716 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:40:42.720 INFO:teuthology.orchestra.run.smithi045.stdout:506 Cannot talk to daemon 2024-09-21T09:40:42.735 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:40:42.750 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:40:42.775 INFO:teuthology.orchestra.run.smithi032.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:40:42.779 INFO:teuthology.orchestra.run.smithi032.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:40:42.779 INFO:teuthology.orchestra.run.smithi032.stdout:=============================================================================== 2024-09-21T09:40:42.779 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:40:42.779 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:40:42.779 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:40:42.779 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:40:42.806 INFO:teuthology.orchestra.run.smithi045.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:40:42.809 INFO:teuthology.orchestra.run.smithi045.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:40:42.810 INFO:teuthology.orchestra.run.smithi045.stdout:=============================================================================== 2024-09-21T09:40:42.810 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:40:42.810 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:40:42.810 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:40:42.810 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:40:42.811 INFO:teuthology.run_tasks:Running task pexec... 2024-09-21T09:40:42.819 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-21T09:40:42.819 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi032.front.sepia.ceph.com 2024-09-21T09:40:42.820 DEBUG:teuthology.orchestra.run.smithi032:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:40:42.820 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi045.front.sepia.ceph.com 2024-09-21T09:40:42.820 DEBUG:teuthology.orchestra.run.smithi045:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:40:43.186 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:40:43.196 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:40:43.196 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-09-21T09:40:43.196 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:40:43.196 INFO:teuthology.orchestra.run.smithi032.stdout:Removing: 2024-09-21T09:40:43.196 INFO:teuthology.orchestra.run.smithi032.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:40:43.196 INFO:teuthology.orchestra.run.smithi032.stdout:Removing unused dependencies: 2024-09-21T09:40:43.196 INFO:teuthology.orchestra.run.smithi032.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:40:43.196 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:40:43.196 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:40:43.197 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:40:43.197 INFO:teuthology.orchestra.run.smithi032.stdout:Remove 2 Packages 2024-09-21T09:40:43.197 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:40:43.197 INFO:teuthology.orchestra.run.smithi032.stdout:Freed space: 5.5 M 2024-09-21T09:40:43.197 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:40:43.201 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:40:43.201 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:40:43.217 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:40:43.227 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:40:43.227 INFO:teuthology.orchestra.run.smithi045.stdout: Package Architecture Version Repository Size 2024-09-21T09:40:43.227 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:40:43.227 INFO:teuthology.orchestra.run.smithi045.stdout:Removing: 2024-09-21T09:40:43.227 INFO:teuthology.orchestra.run.smithi045.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:40:43.228 INFO:teuthology.orchestra.run.smithi045.stdout:Removing unused dependencies: 2024-09-21T09:40:43.228 INFO:teuthology.orchestra.run.smithi045.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:40:43.228 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:40:43.228 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:40:43.228 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:40:43.228 INFO:teuthology.orchestra.run.smithi045.stdout:Remove 2 Packages 2024-09-21T09:40:43.228 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:40:43.228 INFO:teuthology.orchestra.run.smithi045.stdout:Freed space: 5.5 M 2024-09-21T09:40:43.228 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:40:43.232 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:40:43.232 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:40:43.264 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:40:43.264 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:40:43.287 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:40:43.287 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:40:43.401 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:40:43.430 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:40:43.497 INFO:teuthology.orchestra.run.smithi032.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:40:43.517 INFO:teuthology.orchestra.run.smithi045.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:40:43.559 INFO:teuthology.orchestra.run.smithi032.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:40:43.586 INFO:teuthology.orchestra.run.smithi045.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:40:44.054 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:40:44.054 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:40:44.075 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:40:44.075 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:40:44.326 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:40:44.326 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:40:44.326 INFO:teuthology.orchestra.run.smithi045.stdout:Removed: 2024-09-21T09:40:44.327 INFO:teuthology.orchestra.run.smithi045.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:40:44.327 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:40:44.327 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:40:44.403 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:40:44.403 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:40:44.404 INFO:teuthology.orchestra.run.smithi032.stdout:Removed: 2024-09-21T09:40:44.404 INFO:teuthology.orchestra.run.smithi032.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:40:44.404 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:40:44.404 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:40:45.148 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:00:46 ago on Sat 21 Sep 2024 09:39:58 AM UTC. 2024-09-21T09:40:45.283 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: Package Architecture Version Repository Size 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:40:45.284 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:40:45.285 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:40:45.285 INFO:teuthology.orchestra.run.smithi045.stdout:Install 6 Packages 2024-09-21T09:40:45.285 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:40:45.292 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 2.0 M 2024-09-21T09:40:45.292 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 9.2 M 2024-09-21T09:40:45.293 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:40:45.308 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:47 ago on Sat 21 Sep 2024 09:39:58 AM UTC. 2024-09-21T09:40:45.406 INFO:teuthology.orchestra.run.smithi045.stdout:(1/6): nvmetcli-0.7-3.el9.noarch.rpm 489 kB/s | 44 kB 00:00 2024-09-21T09:40:45.432 INFO:teuthology.orchestra.run.smithi045.stdout:(2/6): libnvme-1.9-3.el9.x86_64.rpm 911 kB/s | 106 kB 00:00 2024-09-21T09:40:45.449 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:40:45.450 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:40:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-09-21T09:40:45.450 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:40:45.450 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:40:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:40:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:40:45.450 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:40:45.450 INFO:teuthology.orchestra.run.smithi032.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:40:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:40:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:40:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:40:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:40:45.451 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:40:45.451 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:40:45.451 INFO:teuthology.orchestra.run.smithi032.stdout:Install 6 Packages 2024-09-21T09:40:45.451 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:40:45.465 INFO:teuthology.orchestra.run.smithi045.stdout:(3/6): python3-configshell-1.1.30-1.el9.noarch. 1.2 MB/s | 72 kB 00:00 2024-09-21T09:40:45.467 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 2.0 M 2024-09-21T09:40:45.467 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 9.2 M 2024-09-21T09:40:45.467 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:40:45.491 INFO:teuthology.orchestra.run.smithi045.stdout:(4/6): python3-kmod-0.9-32.el9.x86_64.rpm 1.4 MB/s | 84 kB 00:00 2024-09-21T09:40:45.533 INFO:teuthology.orchestra.run.smithi045.stdout:(5/6): nvme-cli-2.9.1-6.el9.x86_64.rpm 4.1 MB/s | 906 kB 00:00 2024-09-21T09:40:45.641 INFO:teuthology.orchestra.run.smithi045.stdout:(6/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 4.7 MB/s | 837 kB 00:00 2024-09-21T09:40:45.641 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:45.641 INFO:teuthology.orchestra.run.smithi045.stdout:Total 5.7 MB/s | 2.0 MB 00:00 2024-09-21T09:40:45.717 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:40:45.732 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:40:45.732 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:40:45.840 INFO:teuthology.orchestra.run.smithi032.stdout:(1/6): nvmetcli-0.7-3.el9.noarch.rpm 394 kB/s | 44 kB 00:00 2024-09-21T09:40:45.866 INFO:teuthology.orchestra.run.smithi032.stdout:(2/6): libnvme-1.9-3.el9.x86_64.rpm 767 kB/s | 106 kB 00:00 2024-09-21T09:40:45.916 INFO:teuthology.orchestra.run.smithi032.stdout:(3/6): python3-configshell-1.1.30-1.el9.noarch. 952 kB/s | 72 kB 00:00 2024-09-21T09:40:45.941 INFO:teuthology.orchestra.run.smithi032.stdout:(4/6): python3-kmod-0.9-32.el9.x86_64.rpm 1.1 MB/s | 84 kB 00:00 2024-09-21T09:40:46.000 INFO:teuthology.orchestra.run.smithi032.stdout:(5/6): nvme-cli-2.9.1-6.el9.x86_64.rpm 3.2 MB/s | 906 kB 00:00 2024-09-21T09:40:46.011 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:40:46.011 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:40:46.217 INFO:teuthology.orchestra.run.smithi032.stdout:(6/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 2.7 MB/s | 837 kB 00:00 2024-09-21T09:40:46.217 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:46.217 INFO:teuthology.orchestra.run.smithi032.stdout:Total 2.7 MB/s | 2.0 MB 00:00 2024-09-21T09:40:46.295 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:40:46.310 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:40:46.310 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:40:46.451 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:40:46.529 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/6 2024-09-21T09:40:46.562 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/6 2024-09-21T09:40:46.586 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:40:46.586 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:40:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/6 2024-09-21T09:40:46.818 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/6 2024-09-21T09:40:46.847 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-21T09:40:47.034 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:40:47.146 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/6 2024-09-21T09:40:47.180 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/6 2024-09-21T09:40:47.208 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/6 2024-09-21T09:40:47.323 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-21T09:40:47.350 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/6 2024-09-21T09:40:47.435 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/6 2024-09-21T09:40:47.472 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-21T09:40:47.956 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-09-21T09:40:47.984 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/6 2024-09-21T09:40:48.420 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/6 2024-09-21T09:40:48.420 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/6 2024-09-21T09:40:48.420 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/6 2024-09-21T09:40:48.420 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/6 2024-09-21T09:40:48.420 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/6 2024-09-21T09:40:48.420 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/6 2024-09-21T09:40:48.763 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/6 2024-09-21T09:40:48.763 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:40:48.763 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:40:48.763 INFO:teuthology.orchestra.run.smithi045.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:40:48.763 INFO:teuthology.orchestra.run.smithi045.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:40:48.763 INFO:teuthology.orchestra.run.smithi045.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:40:48.763 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:40:48.763 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:40:48.986 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/6 2024-09-21T09:40:48.986 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/6 2024-09-21T09:40:48.986 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/6 2024-09-21T09:40:48.986 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/6 2024-09-21T09:40:48.986 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/6 2024-09-21T09:40:48.986 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/6 2024-09-21T09:40:49.078 DEBUG:teuthology.parallel:result is None 2024-09-21T09:40:49.380 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/6 2024-09-21T09:40:49.380 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:40:49.381 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:40:49.381 INFO:teuthology.orchestra.run.smithi032.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:40:49.381 INFO:teuthology.orchestra.run.smithi032.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:40:49.381 INFO:teuthology.orchestra.run.smithi032.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:40:49.381 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:40:49.381 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:40:49.739 DEBUG:teuthology.parallel:result is None 2024-09-21T09:40:49.739 INFO:teuthology.run_tasks:Running task install... 2024-09-21T09:40:49.747 DEBUG:teuthology.task.install:project ceph 2024-09-21T09:40:49.748 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:40:49.748 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:40:49.748 INFO:teuthology.task.install:Using flavor: default 2024-09-21T09:40:49.753 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T09:40:49.753 INFO:teuthology.task.install:extra packages: [] 2024-09-21T09:40:49.753 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:40:49.753 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:40:49.753 INFO:teuthology.packaging:ref: None 2024-09-21T09:40:49.753 INFO:teuthology.packaging:tag: None 2024-09-21T09:40:49.753 INFO:teuthology.packaging:branch: reef 2024-09-21T09:40:49.753 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:40:49.754 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:40:49.755 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:40:49.755 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:40:49.755 INFO:teuthology.packaging:ref: None 2024-09-21T09:40:49.755 INFO:teuthology.packaging:tag: None 2024-09-21T09:40:49.755 INFO:teuthology.packaging:branch: reef 2024-09-21T09:40:49.755 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:40:49.755 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:40:49.989 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/ 2024-09-21T09:40:49.989 INFO:teuthology.task.install.rpm:Package version is 18.2.4-871.g6a1bdf85 2024-09-21T09:40:50.028 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/ 2024-09-21T09:40:50.028 INFO:teuthology.task.install.rpm:Package version is 18.2.4-871.g6a1bdf85 2024-09-21T09:40:50.097 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:40:50.097 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:40:50.098 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:40:50.130 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-21T09:40:50.130 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:40:50.131 INFO:teuthology.packaging:ref: None 2024-09-21T09:40:50.131 INFO:teuthology.packaging:tag: None 2024-09-21T09:40:50.131 INFO:teuthology.packaging:branch: reef 2024-09-21T09:40:50.131 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:40:50.131 DEBUG:teuthology.orchestra.run.smithi032:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:40:50.135 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:40:50.135 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:40:50.135 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:40:50.169 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-21T09:40:50.170 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:40:50.170 INFO:teuthology.packaging:ref: None 2024-09-21T09:40:50.170 INFO:teuthology.packaging:tag: None 2024-09-21T09:40:50.170 INFO:teuthology.packaging:branch: reef 2024-09-21T09:40:50.170 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:40:50.170 DEBUG:teuthology.orchestra.run.smithi045:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:40:50.201 DEBUG:teuthology.orchestra.run.smithi032:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:40:50.241 DEBUG:teuthology.orchestra.run.smithi045:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:40:50.282 DEBUG:teuthology.orchestra.run.smithi032:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:40:50.322 DEBUG:teuthology.orchestra.run.smithi045:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:40:50.348 INFO:teuthology.orchestra.run.smithi032.stdout:check_obsoletes = 1 2024-09-21T09:40:50.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum clean all 2024-09-21T09:40:50.388 INFO:teuthology.orchestra.run.smithi045.stdout:check_obsoletes = 1 2024-09-21T09:40:50.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum clean all 2024-09-21T09:40:50.591 INFO:teuthology.orchestra.run.smithi032.stdout:45 files removed 2024-09-21T09:40:50.619 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-radosgw 2024-09-21T09:40:50.642 INFO:teuthology.orchestra.run.smithi045.stdout:45 files removed 2024-09-21T09:40:50.669 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-radosgw 2024-09-21T09:40:51.189 INFO:teuthology.orchestra.run.smithi032.stdout:ceph packages for x86_64 285 kB/s | 77 kB 00:00 2024-09-21T09:40:51.248 INFO:teuthology.orchestra.run.smithi045.stdout:ceph packages for x86_64 277 kB/s | 77 kB 00:00 2024-09-21T09:40:51.489 INFO:teuthology.orchestra.run.smithi032.stdout:ceph noarch packages 45 kB/s | 11 kB 00:00 2024-09-21T09:40:51.532 INFO:teuthology.orchestra.run.smithi045.stdout:ceph noarch packages 49 kB/s | 11 kB 00:00 2024-09-21T09:40:51.764 INFO:teuthology.orchestra.run.smithi032.stdout:ceph source packages 8.0 kB/s | 1.9 kB 00:00 2024-09-21T09:40:51.791 INFO:teuthology.orchestra.run.smithi045.stdout:ceph source packages 8.6 kB/s | 1.9 kB 00:00 2024-09-21T09:40:52.492 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.3 MB 00:00 2024-09-21T09:40:52.540 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.3 MB 00:00 2024-09-21T09:40:54.524 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - AppStream 23 MB/s | 20 MB 00:00 2024-09-21T09:40:55.542 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - AppStream 11 MB/s | 20 MB 00:01 2024-09-21T09:41:00.067 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - CRB 4.1 MB/s | 6.5 MB 00:01 2024-09-21T09:41:01.061 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - CRB 4.1 MB/s | 6.5 MB 00:01 2024-09-21T09:41:01.567 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - Extras packages 93 kB/s | 19 kB 00:00 2024-09-21T09:41:02.347 INFO:teuthology.orchestra.run.smithi032.stdout:Extra Packages for Enterprise Linux 35 MB/s | 23 MB 00:00 2024-09-21T09:41:02.669 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - Extras packages 64 kB/s | 19 kB 00:00 2024-09-21T09:41:03.566 INFO:teuthology.orchestra.run.smithi045.stdout:Extra Packages for Enterprise Linux 29 MB/s | 23 MB 00:00 2024-09-21T09:41:08.036 INFO:teuthology.orchestra.run.smithi032.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-09-21T09:41:09.213 INFO:teuthology.orchestra.run.smithi045.stdout:lab-extras 38 kB/s | 1.7 kB 00:00 2024-09-21T09:41:10.339 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:41:10.341 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-radosgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 7.7 M 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout:Upgrading: 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout: librados2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.3 M 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout: librbd1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-base x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 5.1 M 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 18 M 2024-09-21T09:41:10.342 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-selinux x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 25 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: libcephfs2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 711 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: libradosstriper1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 477 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: librgw2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.5 M 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:41:10.343 INFO:teuthology.orchestra.run.smithi032.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ceph-argparse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 45 k 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 129 k 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cephfs x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 161 k 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rados x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 320 k 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 300 k 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 99 k 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout:Install 28 Packages 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout:Upgrade 2 Packages 2024-09-21T09:41:10.344 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:10.345 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 53 M 2024-09-21T09:41:10.345 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:41:10.912 INFO:teuthology.orchestra.run.smithi032.stdout:(1/30): ceph-base-18.2.4-871.g6a1bdf85.el9.x86_ 9.9 MB/s | 5.1 MB 00:00 2024-09-21T09:41:10.963 INFO:teuthology.orchestra.run.smithi032.stdout:(2/30): ceph-selinux-18.2.4-871.g6a1bdf85.el9.x 503 kB/s | 25 kB 00:00 2024-09-21T09:41:11.054 INFO:teuthology.orchestra.run.smithi032.stdout:(3/30): ceph-radosgw-18.2.4-871.g6a1bdf85.el9.x 12 MB/s | 7.7 MB 00:00 2024-09-21T09:41:11.096 INFO:teuthology.orchestra.run.smithi032.stdout:(4/30): libcephfs2-18.2.4-871.g6a1bdf85.el9.x86 5.2 MB/s | 711 kB 00:00 2024-09-21T09:41:11.122 INFO:teuthology.orchestra.run.smithi032.stdout:(5/30): libradosstriper1-18.2.4-871.g6a1bdf85.e 7.0 MB/s | 477 kB 00:00 2024-09-21T09:41:11.172 INFO:teuthology.orchestra.run.smithi032.stdout:(6/30): python3-ceph-argparse-18.2.4-871.g6a1bd 898 kB/s | 45 kB 00:00 2024-09-21T09:41:11.239 INFO:teuthology.orchestra.run.smithi032.stdout:(7/30): python3-ceph-common-18.2.4-871.g6a1bdf8 1.9 MB/s | 129 kB 00:00 2024-09-21T09:41:11.306 INFO:teuthology.orchestra.run.smithi032.stdout:(8/30): python3-cephfs-18.2.4-871.g6a1bdf85.el9 2.4 MB/s | 161 kB 00:00 2024-09-21T09:41:11.373 INFO:teuthology.orchestra.run.smithi032.stdout:(9/30): librgw2-18.2.4-871.g6a1bdf85.el9.x86_64 16 MB/s | 4.5 MB 00:00 2024-09-21T09:41:11.398 INFO:teuthology.orchestra.run.smithi032.stdout:(10/30): python3-rados-18.2.4-871.g6a1bdf85.el9 3.4 MB/s | 320 kB 00:00 2024-09-21T09:41:11.423 INFO:teuthology.orchestra.run.smithi032.stdout:(11/30): python3-rbd-18.2.4-871.g6a1bdf85.el9.x 5.9 MB/s | 300 kB 00:00 2024-09-21T09:41:11.448 INFO:teuthology.orchestra.run.smithi032.stdout:(12/30): python3-rgw-18.2.4-871.g6a1bdf85.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-21T09:41:11.538 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:41:11.540 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-radosgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 7.7 M 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout:Upgrading: 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout: librados2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.3 M 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout: librbd1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-base x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 5.1 M 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 18 M 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-selinux x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 25 k 2024-09-21T09:41:11.541 INFO:teuthology.orchestra.run.smithi045.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: libcephfs2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 711 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: libradosstriper1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 477 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: librgw2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.5 M 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:41:11.542 INFO:teuthology.orchestra.run.smithi045.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ceph-argparse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 45 k 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 129 k 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cephfs x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 161 k 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rados x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 320 k 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 300 k 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 99 k 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout:Install 28 Packages 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout:Upgrade 2 Packages 2024-09-21T09:41:11.543 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:11.544 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 53 M 2024-09-21T09:41:11.544 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:41:11.632 INFO:teuthology.orchestra.run.smithi032.stdout:(13/30): ceph-common-18.2.4-871.g6a1bdf85.el9.x 15 MB/s | 18 MB 00:01 2024-09-21T09:41:12.007 INFO:teuthology.orchestra.run.smithi032.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 183 kB/s | 107 kB 00:00 2024-09-21T09:41:12.033 INFO:teuthology.orchestra.run.smithi032.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 113 kB/s | 45 kB 00:00 2024-09-21T09:41:12.100 INFO:teuthology.orchestra.run.smithi032.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 246 kB/s | 160 kB 00:00 2024-09-21T09:41:12.208 INFO:teuthology.orchestra.run.smithi032.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 391 kB/s | 42 kB 00:00 2024-09-21T09:41:12.250 INFO:teuthology.orchestra.run.smithi032.stdout:(18/30): fmt-8.1.1-5.el9.x86_64.rpm 2.6 MB/s | 111 kB 00:00 2024-09-21T09:41:12.275 INFO:teuthology.orchestra.run.smithi032.stdout:(19/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 12 MB/s | 308 kB 00:00 2024-09-21T09:41:12.318 INFO:teuthology.orchestra.run.smithi032.stdout:(20/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.0 MB/s | 292 kB 00:00 2024-09-21T09:41:12.359 INFO:teuthology.orchestra.run.smithi032.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 627 kB/s | 26 kB 00:00 2024-09-21T09:41:12.384 INFO:teuthology.orchestra.run.smithi032.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-21T09:41:12.410 INFO:teuthology.orchestra.run.smithi032.stdout:(23/30): libunwind-1.6.2-1.el9.x86_64.rpm 2.6 MB/s | 67 kB 00:00 2024-09-21T09:41:12.493 INFO:teuthology.orchestra.run.smithi032.stdout:(24/30): libarrow-9.0.0-11.el9.x86_64.rpm 20 MB/s | 4.4 MB 00:00 2024-09-21T09:41:12.519 INFO:teuthology.orchestra.run.smithi032.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 7.5 MB/s | 191 kB 00:00 2024-09-21T09:41:12.552 INFO:teuthology.orchestra.run.smithi032.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 5.8 MB/s | 839 kB 00:00 2024-09-21T09:41:12.602 INFO:teuthology.orchestra.run.smithi032.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 19 MB/s | 1.6 MB 00:00 2024-09-21T09:41:12.627 INFO:teuthology.orchestra.run.smithi032.stdout:(28/30): librdkafka-1.6.1-102.el9.x86_64.rpm 1.0 MB/s | 662 kB 00:00 2024-09-21T09:41:12.698 INFO:teuthology.orchestra.run.smithi045.stdout:(1/30): ceph-base-18.2.4-871.g6a1bdf85.el9.x86_ 10 MB/s | 5.1 MB 00:00 2024-09-21T09:41:12.773 INFO:teuthology.orchestra.run.smithi045.stdout:(2/30): ceph-selinux-18.2.4-871.g6a1bdf85.el9.x 336 kB/s | 25 kB 00:00 2024-09-21T09:41:12.886 INFO:teuthology.orchestra.run.smithi032.stdout:(29/30): librados2-18.2.4-871.g6a1bdf85.el9.x86 9.9 MB/s | 3.3 MB 00:00 2024-09-21T09:41:12.915 INFO:teuthology.orchestra.run.smithi045.stdout:(3/30): ceph-radosgw-18.2.4-871.g6a1bdf85.el9.x 11 MB/s | 7.7 MB 00:00 2024-09-21T09:41:12.949 INFO:teuthology.orchestra.run.smithi045.stdout:(4/30): libcephfs2-18.2.4-871.g6a1bdf85.el9.x86 4.0 MB/s | 711 kB 00:00 2024-09-21T09:41:12.953 INFO:teuthology.orchestra.run.smithi032.stdout:(30/30): librbd1-18.2.4-871.g6a1bdf85.el9.x86_6 8.6 MB/s | 3.0 MB 00:00 2024-09-21T09:41:12.957 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:12.957 INFO:teuthology.orchestra.run.smithi032.stdout:Total 20 MB/s | 53 MB 00:02 2024-09-21T09:41:12.974 INFO:teuthology.orchestra.run.smithi045.stdout:(5/30): libradosstriper1-18.2.4-871.g6a1bdf85.e 8.0 MB/s | 477 kB 00:00 2024-09-21T09:41:13.024 INFO:teuthology.orchestra.run.smithi045.stdout:(6/30): python3-ceph-argparse-18.2.4-871.g6a1bd 902 kB/s | 45 kB 00:00 2024-09-21T09:41:13.032 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:41:13.071 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:41:13.071 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:41:13.091 INFO:teuthology.orchestra.run.smithi045.stdout:(7/30): python3-ceph-common-18.2.4-871.g6a1bdf8 1.9 MB/s | 129 kB 00:00 2024-09-21T09:41:13.158 INFO:teuthology.orchestra.run.smithi045.stdout:(8/30): python3-cephfs-18.2.4-871.g6a1bdf85.el9 2.4 MB/s | 161 kB 00:00 2024-09-21T09:41:13.367 INFO:teuthology.orchestra.run.smithi045.stdout:(9/30): ceph-common-18.2.4-871.g6a1bdf85.el9.x8 16 MB/s | 18 MB 00:01 2024-09-21T09:41:13.434 INFO:teuthology.orchestra.run.smithi045.stdout:(10/30): librgw2-18.2.4-871.g6a1bdf85.el9.x86_6 9.2 MB/s | 4.5 MB 00:00 2024-09-21T09:41:13.460 INFO:teuthology.orchestra.run.smithi045.stdout:(11/30): python3-rados-18.2.4-871.g6a1bdf85.el9 1.0 MB/s | 320 kB 00:00 2024-09-21T09:41:13.485 INFO:teuthology.orchestra.run.smithi045.stdout:(12/30): python3-rgw-18.2.4-871.g6a1bdf85.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-21T09:41:13.510 INFO:teuthology.orchestra.run.smithi045.stdout:(13/30): python3-rbd-18.2.4-871.g6a1bdf85.el9.x 2.1 MB/s | 300 kB 00:00 2024-09-21T09:41:13.544 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:41:13.544 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:41:14.219 INFO:teuthology.orchestra.run.smithi045.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 141 kB/s | 107 kB 00:00 2024-09-21T09:41:14.244 INFO:teuthology.orchestra.run.smithi045.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 62 kB/s | 45 kB 00:00 2024-09-21T09:41:14.269 INFO:teuthology.orchestra.run.smithi045.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 204 kB/s | 160 kB 00:00 2024-09-21T09:41:14.844 INFO:teuthology.orchestra.run.smithi045.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 73 kB/s | 42 kB 00:00 2024-09-21T09:41:14.886 INFO:teuthology.orchestra.run.smithi045.stdout:(18/30): fmt-8.1.1-5.el9.x86_64.rpm 2.6 MB/s | 111 kB 00:00 2024-09-21T09:41:14.920 INFO:teuthology.orchestra.run.smithi045.stdout:(19/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 9.1 MB/s | 308 kB 00:00 2024-09-21T09:41:15.090 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:41:15.154 INFO:teuthology.orchestra.run.smithi045.stdout:(20/30): libarrow-9.0.0-11.el9.x86_64.rpm 19 MB/s | 4.4 MB 00:00 2024-09-21T09:41:15.171 INFO:teuthology.orchestra.run.smithi045.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.6 MB/s | 26 kB 00:00 2024-09-21T09:41:15.196 INFO:teuthology.orchestra.run.smithi045.stdout:(22/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 307 kB/s | 292 kB 00:00 2024-09-21T09:41:15.213 INFO:teuthology.orchestra.run.smithi045.stdout:(23/30): liboath-2.6.7-2.el9.x86_64.rpm 1.1 MB/s | 49 kB 00:00 2024-09-21T09:41:15.255 INFO:teuthology.orchestra.run.smithi045.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2024-09-21T09:41:15.280 INFO:teuthology.orchestra.run.smithi045.stdout:(25/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 14 MB/s | 839 kB 00:00 2024-09-21T09:41:15.300 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-21T09:41:15.314 INFO:teuthology.orchestra.run.smithi045.stdout:(26/30): re2-20211101-20.el9.x86_64.rpm 3.2 MB/s | 191 kB 00:00 2024-09-21T09:41:15.341 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-21T09:41:15.373 INFO:teuthology.orchestra.run.smithi045.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 17 MB/s | 1.6 MB 00:00 2024-09-21T09:41:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-21T09:41:15.406 INFO:teuthology.orchestra.run.smithi045.stdout:(28/30): librdkafka-1.6.1-102.el9.x86_64.rpm 558 kB/s | 662 kB 00:01 2024-09-21T09:41:15.565 INFO:teuthology.orchestra.run.smithi045.stdout:(29/30): librados2-18.2.4-871.g6a1bdf85.el9.x86 13 MB/s | 3.3 MB 00:00 2024-09-21T09:41:15.657 INFO:teuthology.orchestra.run.smithi045.stdout:(30/30): librbd1-18.2.4-871.g6a1bdf85.el9.x86_6 11 MB/s | 3.0 MB 00:00 2024-09-21T09:41:15.662 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:15.662 INFO:teuthology.orchestra.run.smithi045.stdout:Total 13 MB/s | 53 MB 00:04 2024-09-21T09:41:15.737 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:41:15.778 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:41:15.778 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:41:15.780 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-21T09:41:15.823 INFO:teuthology.orchestra.run.smithi032.stdout: Upgrading : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:41:15.923 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:41:16.014 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:41:16.166 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:41:16.230 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:41:16.250 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:41:16.251 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:41:16.266 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-21T09:41:16.310 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 9/32 2024-09-21T09:41:16.378 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:41:16.401 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:41:16.468 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:41:16.577 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-21T09:41:16.610 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-21T09:41:16.659 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-21T09:41:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-21T09:41:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-21T09:41:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-21T09:41:16.831 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-21T09:41:17.170 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-21T09:41:17.341 INFO:teuthology.orchestra.run.smithi032.stdout: Upgrading : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:41:17.433 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:41:17.463 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 21/32 2024-09-21T09:41:17.498 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-21T09:41:17.636 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:41:17.722 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-21T09:41:17.770 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-21T09:41:17.822 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-21T09:41:17.925 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-21T09:41:17.952 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:41:18.024 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:41:18.102 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 25/32 2024-09-21T09:41:18.137 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 26/32 2024-09-21T09:41:18.203 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-21T09:41:18.303 INFO:teuthology.orchestra.run.smithi045.stdout: Upgrading : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:41:18.379 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:41:18.466 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:41:18.488 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:41:18.542 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:41:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-21T09:41:18.624 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 9/32 2024-09-21T09:41:18.700 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:41:18.723 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:41:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:41:18.901 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-21T09:41:18.932 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-21T09:41:18.990 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-21T09:41:19.030 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-21T09:41:19.062 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-21T09:41:19.107 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-21T09:41:19.149 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-21T09:41:19.473 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-21T09:41:19.511 INFO:teuthology.orchestra.run.smithi045.stdout: Upgrading : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:41:19.602 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:41:19.636 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 21/32 2024-09-21T09:41:19.837 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-21T09:41:20.263 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-21T09:41:20.322 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:41:20.330 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:41:20.359 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:41:20.402 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:41:20.483 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 25/32 2024-09-21T09:41:20.516 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 26/32 2024-09-21T09:41:20.958 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:41:21.004 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:41:21.104 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:41:21.104 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:41:21.104 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:41:21.104 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:21.138 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:41:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:41:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:41:23.310 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:41:23.350 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:41:23.448 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:41:23.448 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:41:23.448 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:41:23.449 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:23.484 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:41:31.603 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:41:31.603 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /sys 2024-09-21T09:41:31.603 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /proc 2024-09-21T09:41:31.603 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /mnt 2024-09-21T09:41:31.603 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /var/tmp 2024-09-21T09:41:31.603 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /home 2024-09-21T09:41:31.603 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /root 2024-09-21T09:41:31.603 INFO:teuthology.orchestra.run.smithi032.stdout:skipping the directory /tmp 2024-09-21T09:41:31.603 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:32.378 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:41:32.405 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:41:32.405 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:32.405 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:41:32.405 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:41:32.406 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:41:32.406 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:32.407 INFO:teuthology.orchestra.run.smithi032.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:41:32.515 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:41:32.515 INFO:teuthology.orchestra.run.smithi032.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 8/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 9/32 2024-09-21T09:41:33.601 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 12/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 13/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-21T09:41:33.602 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-21T09:41:33.603 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-21T09:41:33.603 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-21T09:41:33.603 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-21T09:41:33.603 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-21T09:41:33.603 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-21T09:41:33.603 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:41:33.604 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-21T09:41:33.604 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/32 2024-09-21T09:41:33.654 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:41:33.654 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /sys 2024-09-21T09:41:33.654 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /proc 2024-09-21T09:41:33.654 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /mnt 2024-09-21T09:41:33.654 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /var/tmp 2024-09-21T09:41:33.654 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /home 2024-09-21T09:41:33.654 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /root 2024-09-21T09:41:33.654 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /tmp 2024-09-21T09:41:33.654 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:34.394 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout:Upgraded: 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-21T09:41:34.395 INFO:teuthology.orchestra.run.smithi032.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.396 INFO:teuthology.orchestra.run.smithi032.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:41:34.397 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.397 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.397 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:34.397 INFO:teuthology.orchestra.run.smithi032.stdout: re2-1:20211101-20.el9.x86_64 2024-09-21T09:41:34.397 INFO:teuthology.orchestra.run.smithi032.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:41:34.397 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:34.397 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:41:34.444 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:41:34.472 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:41:34.472 INFO:teuthology.orchestra.run.smithi045.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:34.472 INFO:teuthology.orchestra.run.smithi045.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:41:34.472 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:41:34.472 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:41:34.472 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:34.475 INFO:teuthology.orchestra.run.smithi045.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:41:34.564 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:41:34.565 INFO:teuthology.orchestra.run.smithi045.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:41:34.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-test 2024-09-21T09:41:35.356 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:27 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:41:35.505 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:41:35.506 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:35.506 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-09-21T09:41:35.506 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:35.506 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:41:35.506 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-test x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 40 M 2024-09-21T09:41:35.506 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:41:35.506 INFO:teuthology.orchestra.run.smithi032.stdout: libcephsqlite x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 166 k 2024-09-21T09:41:35.506 INFO:teuthology.orchestra.run.smithi032.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:41:35.506 INFO:teuthology.orchestra.run.smithi032.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:41:35.507 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:35.507 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:41:35.507 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:35.507 INFO:teuthology.orchestra.run.smithi032.stdout:Install 4 Packages 2024-09-21T09:41:35.507 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:35.507 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 40 M 2024-09-21T09:41:35.507 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 164 M 2024-09-21T09:41:35.508 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 8/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 9/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:41:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 12/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 13/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-21T09:41:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-21T09:41:35.695 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-21T09:41:35.695 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-21T09:41:35.695 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-21T09:41:35.695 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-21T09:41:35.695 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:41:35.695 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-21T09:41:35.695 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/32 2024-09-21T09:41:35.905 INFO:teuthology.orchestra.run.smithi032.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 2.5 MB/s | 304 kB 00:00 2024-09-21T09:41:35.947 INFO:teuthology.orchestra.run.smithi032.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-09-21T09:41:35.989 INFO:teuthology.orchestra.run.smithi032.stdout:(3/4): libcephsqlite-18.2.4-871.g6a1bdf85.el9.x 815 kB/s | 166 kB 00:00 2024-09-21T09:41:36.696 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout:Upgraded: 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:41:36.697 INFO:teuthology.orchestra.run.smithi045.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.698 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.699 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.699 INFO:teuthology.orchestra.run.smithi045.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:41:36.699 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.699 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.699 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:36.699 INFO:teuthology.orchestra.run.smithi045.stdout: re2-1:20211101-20.el9.x86_64 2024-09-21T09:41:36.699 INFO:teuthology.orchestra.run.smithi045.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:41:36.699 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:36.699 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:41:37.065 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-test 2024-09-21T09:41:37.088 INFO:teuthology.orchestra.run.smithi032.stdout:(4/4): ceph-test-18.2.4-871.g6a1bdf85.el9.x86_6 30 MB/s | 40 MB 00:01 2024-09-21T09:41:37.090 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:37.091 INFO:teuthology.orchestra.run.smithi032.stdout:Total 25 MB/s | 40 MB 00:01 2024-09-21T09:41:37.119 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:41:37.141 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:41:37.141 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:41:37.477 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:41:37.478 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:41:37.669 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:00:28 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:41:37.807 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-test x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 40 M 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout: libcephsqlite x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 166 k 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:41:37.808 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:37.809 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:41:37.809 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:41:37.809 INFO:teuthology.orchestra.run.smithi045.stdout:Install 4 Packages 2024-09-21T09:41:37.809 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:37.809 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 40 M 2024-09-21T09:41:37.809 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 164 M 2024-09-21T09:41:37.809 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:41:37.971 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:41:38.053 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-21T09:41:38.097 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-21T09:41:38.132 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:41:38.303 INFO:teuthology.orchestra.run.smithi045.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.4 MB/s | 304 kB 00:00 2024-09-21T09:41:38.328 INFO:teuthology.orchestra.run.smithi045.stdout:(2/4): libcephsqlite-18.2.4-871.g6a1bdf85.el9.x 689 kB/s | 166 kB 00:00 2024-09-21T09:41:38.354 INFO:teuthology.orchestra.run.smithi045.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-09-21T09:41:39.346 INFO:teuthology.orchestra.run.smithi045.stdout:(4/4): ceph-test-18.2.4-871.g6a1bdf85.el9.x86_6 32 MB/s | 40 MB 00:01 2024-09-21T09:41:39.348 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:39.348 INFO:teuthology.orchestra.run.smithi045.stdout:Total 26 MB/s | 40 MB 00:01 2024-09-21T09:41:39.377 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:41:39.399 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:41:39.399 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:41:39.735 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:41:39.736 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:41:40.245 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:41:40.340 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-21T09:41:40.396 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-21T09:41:40.423 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:41:42.049 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:41:42.133 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:41:44.158 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:41:44.158 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/4 2024-09-21T09:41:44.158 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/4 2024-09-21T09:41:44.158 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-21T09:41:44.349 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:41:44.436 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:41:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-21T09:41:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:44.497 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:41:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:41:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:41:44.497 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:44.497 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:41:44.816 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph 2024-09-21T09:41:45.422 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:37 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:41:45.574 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:41:45.576 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:45.576 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-09-21T09:41:45.576 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: ceph x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 6.5 k 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mds x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 2.1 M 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 1.5 M 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-modules-core noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 246 k 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mon x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.7 M 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-osd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 17 M 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:41:45.577 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:41:45.578 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout:Install 33 Packages 2024-09-21T09:41:45.579 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:45.580 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 29 M 2024-09-21T09:41:45.580 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 104 M 2024-09-21T09:41:45.580 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:41:46.239 INFO:teuthology.orchestra.run.smithi032.stdout:(1/33): ceph-18.2.4-871.g6a1bdf85.el9.x86_64.rp 49 kB/s | 6.5 kB 00:00 2024-09-21T09:41:46.409 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:41:46.409 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/4 2024-09-21T09:41:46.409 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/4 2024-09-21T09:41:46.409 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-21T09:41:46.456 INFO:teuthology.orchestra.run.smithi032.stdout:(2/33): ceph-mds-18.2.4-871.g6a1bdf85.el9.x86_6 6.0 MB/s | 2.1 MB 00:00 2024-09-21T09:41:46.498 INFO:teuthology.orchestra.run.smithi032.stdout:(3/33): ceph-mgr-18.2.4-871.g6a1bdf85.el9.x86_6 3.7 MB/s | 1.5 MB 00:00 2024-09-21T09:41:46.615 INFO:teuthology.orchestra.run.smithi032.stdout:(4/33): ceph-mon-18.2.4-871.g6a1bdf85.el9.x86_6 12 MB/s | 4.7 MB 00:00 2024-09-21T09:41:46.640 INFO:teuthology.orchestra.run.smithi032.stdout:(5/33): ceph-mgr-modules-core-18.2.4-871.g6a1bd 1.7 MB/s | 246 kB 00:00 2024-09-21T09:41:46.712 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-21T09:41:46.712 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:46.712 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:41:46.712 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:46.712 INFO:teuthology.orchestra.run.smithi045.stdout: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:46.713 INFO:teuthology.orchestra.run.smithi045.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:41:46.713 INFO:teuthology.orchestra.run.smithi045.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:41:46.713 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:46.713 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:41:47.007 INFO:teuthology.orchestra.run.smithi032.stdout:(6/33): ceph-osd-18.2.4-871.g6a1bdf85.el9.x86_6 30 MB/s | 17 MB 00:00 2024-09-21T09:41:47.032 INFO:teuthology.orchestra.run.smithi032.stdout:(7/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 607 kB/s | 253 kB 00:00 2024-09-21T09:41:47.064 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph 2024-09-21T09:41:47.132 INFO:teuthology.orchestra.run.smithi032.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 1.3 MB/s | 135 kB 00:00 2024-09-21T09:41:47.174 INFO:teuthology.orchestra.run.smithi032.stdout:(9/33): python3-cryptography-36.0.1-4.el9.x86_6 2.3 MB/s | 1.2 MB 00:00 2024-09-21T09:41:47.208 INFO:teuthology.orchestra.run.smithi032.stdout:(10/33): python3-ply-3.11-14.el9.noarch.rpm 531 kB/s | 106 kB 00:00 2024-09-21T09:41:47.233 INFO:teuthology.orchestra.run.smithi032.stdout:(11/33): python3-requests-2.25.1-8.el9.noarch.r 1.2 MB/s | 125 kB 00:00 2024-09-21T09:41:47.258 INFO:teuthology.orchestra.run.smithi032.stdout:(12/33): python3-urllib3-1.26.5-6.el9.noarch.rp 2.5 MB/s | 215 kB 00:00 2024-09-21T09:41:47.525 INFO:teuthology.orchestra.run.smithi032.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 119 kB/s | 35 kB 00:00 2024-09-21T09:41:47.634 INFO:teuthology.orchestra.run.smithi032.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 136 kB/s | 51 kB 00:00 2024-09-21T09:41:47.666 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:00:38 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:41:47.676 INFO:teuthology.orchestra.run.smithi032.stdout:(15/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.0 MB/s | 43 kB 00:00 2024-09-21T09:41:47.701 INFO:teuthology.orchestra.run.smithi032.stdout:(16/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 6.7 MB/s | 172 kB 00:00 2024-09-21T09:41:47.726 INFO:teuthology.orchestra.run.smithi032.stdout:(17/33): python3-toml-0.10.2-6.el9.noarch.rpm 208 kB/s | 42 kB 00:00 2024-09-21T09:41:47.760 INFO:teuthology.orchestra.run.smithi032.stdout:(18/33): python3-cherrypy-18.6.1-2.el9.noarch.r 6.0 MB/s | 358 kB 00:00 2024-09-21T09:41:47.785 INFO:teuthology.orchestra.run.smithi032.stdout:(19/33): python3-mako-1.1.4-6.el9.noarch.rpm 312 kB/s | 172 kB 00:00 2024-09-21T09:41:47.802 INFO:teuthology.orchestra.run.smithi032.stdout:(20/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 142 kB/s | 11 kB 00:00 2024-09-21T09:41:47.806 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:41:47.808 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:41:47.808 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2024-09-21T09:41:47.808 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: ceph x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 6.5 k 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mds x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 2.1 M 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 1.5 M 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-modules-core noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 246 k 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mon x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.7 M 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-osd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 17 M 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:41:47.809 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:41:47.810 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout:Install 33 Packages 2024-09-21T09:41:47.811 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:47.812 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 29 M 2024-09-21T09:41:47.812 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 104 M 2024-09-21T09:41:47.812 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:41:47.827 INFO:teuthology.orchestra.run.smithi032.stdout:(21/33): python3-jaraco-classes-3.2.1-5.el9.noa 271 kB/s | 18 kB 00:00 2024-09-21T09:41:47.844 INFO:teuthology.orchestra.run.smithi032.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 574 kB/s | 19 kB 00:00 2024-09-21T09:41:47.861 INFO:teuthology.orchestra.run.smithi032.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 627 kB/s | 20 kB 00:00 2024-09-21T09:41:47.878 INFO:teuthology.orchestra.run.smithi032.stdout:(24/33): python3-jaraco-collections-3.0.0-8.el9 250 kB/s | 23 kB 00:00 2024-09-21T09:41:47.895 INFO:teuthology.orchestra.run.smithi032.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 922 kB/s | 46 kB 00:00 2024-09-21T09:41:47.920 INFO:teuthology.orchestra.run.smithi032.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 1.3 MB/s | 79 kB 00:00 2024-09-21T09:41:47.946 INFO:teuthology.orchestra.run.smithi032.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 328 kB/s | 16 kB 00:00 2024-09-21T09:41:47.971 INFO:teuthology.orchestra.run.smithi032.stdout:(28/33): python3-tempora-5.0.0-2.el9.noarch.rpm 1.4 MB/s | 36 kB 00:00 2024-09-21T09:41:47.996 INFO:teuthology.orchestra.run.smithi032.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.2 MB/s | 90 kB 00:00 2024-09-21T09:41:48.021 INFO:teuthology.orchestra.run.smithi032.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.9 MB/s | 272 kB 00:00 2024-09-21T09:41:48.027 INFO:teuthology.orchestra.run.smithi045.stdout:(1/33): ceph-18.2.4-871.g6a1bdf85.el9.x86_64.rp 46 kB/s | 6.5 kB 00:00 2024-09-21T09:41:48.047 INFO:teuthology.orchestra.run.smithi032.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 803 kB/s | 20 kB 00:00 2024-09-21T09:41:48.072 INFO:teuthology.orchestra.run.smithi032.stdout:(32/33): python3-webob-1.8.8-2.el9.noarch.rpm 2.2 MB/s | 230 kB 00:00 2024-09-21T09:41:48.105 INFO:teuthology.orchestra.run.smithi032.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-09-21T09:41:48.110 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:48.110 INFO:teuthology.orchestra.run.smithi032.stdout:Total 12 MB/s | 29 MB 00:02 2024-09-21T09:41:48.228 INFO:teuthology.orchestra.run.smithi045.stdout:(2/33): ceph-mds-18.2.4-871.g6a1bdf85.el9.x86_6 6.1 MB/s | 2.1 MB 00:00 2024-09-21T09:41:48.238 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:41:48.270 INFO:teuthology.orchestra.run.smithi045.stdout:(3/33): ceph-mgr-18.2.4-871.g6a1bdf85.el9.x86_6 3.8 MB/s | 1.5 MB 00:00 2024-09-21T09:41:48.279 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:41:48.279 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:41:48.362 INFO:teuthology.orchestra.run.smithi045.stdout:(4/33): ceph-mgr-modules-core-18.2.4-871.g6a1bd 2.6 MB/s | 246 kB 00:00 2024-09-21T09:41:48.479 INFO:teuthology.orchestra.run.smithi045.stdout:(5/33): ceph-mon-18.2.4-871.g6a1bdf85.el9.x86_6 10 MB/s | 4.7 MB 00:00 2024-09-21T09:41:48.554 INFO:teuthology.orchestra.run.smithi045.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.3 MB/s | 253 kB 00:00 2024-09-21T09:41:48.596 INFO:teuthology.orchestra.run.smithi045.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-09-21T09:41:48.673 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:41:48.673 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:41:48.771 INFO:teuthology.orchestra.run.smithi045.stdout:(8/33): ceph-osd-18.2.4-871.g6a1bdf85.el9.x86_6 30 MB/s | 17 MB 00:00 2024-09-21T09:41:48.797 INFO:teuthology.orchestra.run.smithi045.stdout:(9/33): python3-pycparser-2.20-6.el9.noarch.rpm 674 kB/s | 135 kB 00:00 2024-09-21T09:41:48.838 INFO:teuthology.orchestra.run.smithi045.stdout:(10/33): python3-cryptography-36.0.1-4.el9.x86_ 3.4 MB/s | 1.2 MB 00:00 2024-09-21T09:41:48.863 INFO:teuthology.orchestra.run.smithi045.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 3.2 MB/s | 215 kB 00:00 2024-09-21T09:41:48.905 INFO:teuthology.orchestra.run.smithi045.stdout:(12/33): python3-requests-2.25.1-8.el9.noarch.r 939 kB/s | 125 kB 00:00 2024-09-21T09:41:48.955 INFO:teuthology.orchestra.run.smithi045.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 378 kB/s | 35 kB 00:00 2024-09-21T09:41:48.997 INFO:teuthology.orchestra.run.smithi045.stdout:(14/33): python3-mako-1.1.4-6.el9.noarch.rpm 1.1 MB/s | 172 kB 00:00 2024-09-21T09:41:49.022 INFO:teuthology.orchestra.run.smithi045.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 435 kB/s | 51 kB 00:00 2024-09-21T09:41:49.048 INFO:teuthology.orchestra.run.smithi045.stdout:(16/33): python3-toml-0.10.2-6.el9.noarch.rpm 451 kB/s | 42 kB 00:00 2024-09-21T09:41:49.098 INFO:teuthology.orchestra.run.smithi045.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 431 kB/s | 43 kB 00:00 2024-09-21T09:41:49.123 INFO:teuthology.orchestra.run.smithi045.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 465 kB/s | 11 kB 00:00 2024-09-21T09:41:49.149 INFO:teuthology.orchestra.run.smithi045.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 711 kB/s | 18 kB 00:00 2024-09-21T09:41:49.174 INFO:teuthology.orchestra.run.smithi045.stdout:(20/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.1 MB/s | 172 kB 00:00 2024-09-21T09:41:49.199 INFO:teuthology.orchestra.run.smithi045.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 461 kB/s | 23 kB 00:00 2024-09-21T09:41:49.224 INFO:teuthology.orchestra.run.smithi045.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 388 kB/s | 19 kB 00:00 2024-09-21T09:41:49.250 INFO:teuthology.orchestra.run.smithi045.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 391 kB/s | 20 kB 00:00 2024-09-21T09:41:49.275 INFO:teuthology.orchestra.run.smithi045.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 919 kB/s | 46 kB 00:00 2024-09-21T09:41:49.301 INFO:teuthology.orchestra.run.smithi045.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-21T09:41:49.351 INFO:teuthology.orchestra.run.smithi045.stdout:(26/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.2 MB/s | 358 kB 00:00 2024-09-21T09:41:49.376 INFO:teuthology.orchestra.run.smithi045.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 218 kB/s | 16 kB 00:00 2024-09-21T09:41:49.402 INFO:teuthology.orchestra.run.smithi045.stdout:(28/33): python3-tempora-5.0.0-2.el9.noarch.rpm 1.4 MB/s | 36 kB 00:00 2024-09-21T09:41:49.427 INFO:teuthology.orchestra.run.smithi045.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.2 MB/s | 90 kB 00:00 2024-09-21T09:41:49.461 INFO:teuthology.orchestra.run.smithi045.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.4 MB/s | 272 kB 00:00 2024-09-21T09:41:49.486 INFO:teuthology.orchestra.run.smithi045.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 806 kB/s | 20 kB 00:00 2024-09-21T09:41:49.511 INFO:teuthology.orchestra.run.smithi045.stdout:(32/33): python3-webob-1.8.8-2.el9.noarch.rpm 2.1 MB/s | 230 kB 00:00 2024-09-21T09:41:49.536 INFO:teuthology.orchestra.run.smithi045.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-09-21T09:41:49.541 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:49.541 INFO:teuthology.orchestra.run.smithi045.stdout:Total 17 MB/s | 29 MB 00:01 2024-09-21T09:41:49.669 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:41:49.711 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:41:49.711 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:41:49.883 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:41:49.958 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-21T09:41:49.991 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-21T09:41:50.024 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-21T09:41:50.121 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-21T09:41:50.139 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:41:50.139 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:41:50.197 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-21T09:41:50.252 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-21T09:41:50.295 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-21T09:41:50.333 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-21T09:41:50.367 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-21T09:41:50.417 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-21T09:41:50.458 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-21T09:41:50.502 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-21T09:41:50.535 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-21T09:41:50.574 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-21T09:41:50.652 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-21T09:41:50.811 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-21T09:41:50.902 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-21T09:41:50.961 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-21T09:41:50.998 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-21T09:41:51.058 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-21T09:41:51.121 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-21T09:41:51.320 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-21T09:41:51.369 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-21T09:41:51.386 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:41:51.440 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-21T09:41:51.449 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-21T09:41:51.472 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-21T09:41:51.505 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-21T09:41:51.539 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-21T09:41:51.576 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-21T09:41:51.600 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-21T09:41:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-21T09:41:51.698 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-21T09:41:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-21T09:41:51.776 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-21T09:41:51.806 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-21T09:41:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-21T09:41:51.874 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-21T09:41:51.878 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 28/33 2024-09-21T09:41:51.910 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:41:51.923 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-21T09:41:51.937 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:41:51.937 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:51.937 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:41:51.938 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:41:51.938 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:41:51.938 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:51.975 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-21T09:41:52.009 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-21T09:41:52.048 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-21T09:41:52.132 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-21T09:41:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-21T09:41:52.367 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-21T09:41:52.418 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-21T09:41:52.465 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-21T09:41:52.532 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-21T09:41:52.596 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-21T09:41:52.790 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-21T09:41:52.853 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-21T09:41:52.942 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-21T09:41:53.040 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-21T09:41:53.085 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-21T09:41:53.207 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-21T09:41:53.386 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 28/33 2024-09-21T09:41:53.413 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:41:53.437 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:41:53.440 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:41:53.440 INFO:teuthology.orchestra.run.smithi045.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:53.440 INFO:teuthology.orchestra.run.smithi045.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:41:53.440 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:41:53.440 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:41:53.440 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:53.468 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:41:53.468 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:53.468 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:41:53.468 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:41:53.468 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:41:53.468 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:41:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:41:53.990 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:53.990 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:41:53.990 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:41:53.990 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:41:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:54.207 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:41:54.233 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:41:54.233 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:54.233 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:41:54.233 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:41:54.233 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:41:54.233 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:54.298 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:41:54.947 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:41:54.978 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:41:54.978 INFO:teuthology.orchestra.run.smithi045.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:54.979 INFO:teuthology.orchestra.run.smithi045.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:41:54.979 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:41:54.979 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:41:54.979 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:55.448 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:41:55.475 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:41:55.475 INFO:teuthology.orchestra.run.smithi045.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:55.475 INFO:teuthology.orchestra.run.smithi045.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:41:55.475 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:41:55.475 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:41:55.475 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:55.691 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:41:55.716 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:41:55.716 INFO:teuthology.orchestra.run.smithi045.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:55.716 INFO:teuthology.orchestra.run.smithi045.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:41:55.716 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:41:55.716 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:41:55.716 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:55.782 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 6/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-21T09:41:56.074 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-21T09:41:56.075 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-21T09:41:56.076 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-21T09:41:56.076 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-21T09:41:56.076 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-21T09:41:56.076 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-21T09:41:56.077 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-21T09:41:56.077 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-21T09:41:56.077 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-21T09:41:56.077 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-21T09:41:57.084 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-21T09:41:57.084 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:57.084 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:41:57.084 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:41:57.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-base 2024-09-21T09:41:57.611 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:41:57.611 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 6/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-21T09:41:57.612 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-21T09:41:57.613 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-21T09:41:57.614 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-21T09:41:57.615 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-21T09:41:57.615 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-21T09:41:57.615 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-21T09:41:57.615 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-21T09:41:57.615 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-21T09:41:57.615 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-21T09:41:58.073 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:50 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:41:58.165 INFO:teuthology.orchestra.run.smithi032.stdout:Package ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:41:58.209 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:41:58.210 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-09-21T09:41:58.210 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:41:58.273 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install cephadm 2024-09-21T09:41:58.480 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-21T09:41:58.480 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:58.480 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:41:58.480 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:58.480 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:58.480 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:58.480 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:41:58.480 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:41:58.481 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:41:58.483 INFO:teuthology.orchestra.run.smithi045.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:41:58.484 INFO:teuthology.orchestra.run.smithi045.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:41:58.484 INFO:teuthology.orchestra.run.smithi045.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:41:58.484 INFO:teuthology.orchestra.run.smithi045.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:41:58.484 INFO:teuthology.orchestra.run.smithi045.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:41:58.484 INFO:teuthology.orchestra.run.smithi045.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:41:58.484 INFO:teuthology.orchestra.run.smithi045.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:41:58.484 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:41:58.484 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:41:58.881 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:50 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:41:58.882 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-base 2024-09-21T09:41:59.017 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:41:59.018 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:59.018 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-09-21T09:41:59.018 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:59.018 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:41:59.019 INFO:teuthology.orchestra.run.smithi032.stdout: cephadm noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 226 k 2024-09-21T09:41:59.019 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:59.019 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:41:59.019 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:41:59.019 INFO:teuthology.orchestra.run.smithi032.stdout:Install 1 Package 2024-09-21T09:41:59.019 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:41:59.019 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 226 k 2024-09-21T09:41:59.020 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 217 k 2024-09-21T09:41:59.020 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:41:59.239 INFO:teuthology.orchestra.run.smithi032.stdout:cephadm-18.2.4-871.g6a1bdf85.el9.noarch.rpm 1.0 MB/s | 226 kB 00:00 2024-09-21T09:41:59.239 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:59.239 INFO:teuthology.orchestra.run.smithi032.stdout:Total 1.0 MB/s | 226 kB 00:00 2024-09-21T09:41:59.240 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:41:59.245 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:41:59.245 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:41:59.248 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:41:59.248 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:41:59.320 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:41:59.498 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:00:50 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:41:59.591 INFO:teuthology.orchestra.run.smithi045.stdout:Package ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:41:59.635 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:41:59.636 INFO:teuthology.orchestra.run.smithi045.stdout:Nothing to do. 2024-09-21T09:41:59.636 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:41:59.714 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install cephadm 2024-09-21T09:41:59.787 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:41:59.879 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:42:00.256 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:42:00.319 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:00:51 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:00.455 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout: cephadm noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 226 k 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout:Install 1 Package 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:00.456 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 226 k 2024-09-21T09:42:00.457 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 217 k 2024-09-21T09:42:00.457 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:00.538 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:42:00.539 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:00.539 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:00.539 INFO:teuthology.orchestra.run.smithi032.stdout: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:00.539 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:00.539 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:00.700 INFO:teuthology.orchestra.run.smithi045.stdout:cephadm-18.2.4-871.g6a1bdf85.el9.noarch.rpm 930 kB/s | 226 kB 00:00 2024-09-21T09:42:00.701 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:00.701 INFO:teuthology.orchestra.run.smithi045.stdout:Total 924 kB/s | 226 kB 00:00 2024-09-21T09:42:00.701 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:00.707 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:00.707 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:00.710 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:00.710 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:00.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-immutable-object-cache 2024-09-21T09:42:00.784 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:01.174 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:42:01.274 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:42:01.363 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:53 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:01.502 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:01.503 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:01.503 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repo Size 2024-09-21T09:42:01.503 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:01.503 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:01.503 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 142 k 2024-09-21T09:42:01.503 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:01.503 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:01.504 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:01.504 INFO:teuthology.orchestra.run.smithi032.stdout:Install 1 Package 2024-09-21T09:42:01.504 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:01.504 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 142 k 2024-09-21T09:42:01.504 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 395 k 2024-09-21T09:42:01.504 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:01.632 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:42:01.695 INFO:teuthology.orchestra.run.smithi032.stdout:ceph-immutable-object-cache-18.2.4-871.g6a1bdf8 749 kB/s | 142 kB 00:00 2024-09-21T09:42:01.695 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:01.696 INFO:teuthology.orchestra.run.smithi032.stdout:Total 743 kB/s | 142 kB 00:00 2024-09-21T09:42:01.696 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:01.703 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:01.703 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:01.764 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:01.764 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:01.895 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:42:01.895 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:01.896 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:01.896 INFO:teuthology.orchestra.run.smithi045.stdout: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:01.896 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:01.896 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:01.911 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:01.970 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:42:01.997 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:42:01.997 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:42:01.997 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:42:01.997 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:02.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-immutable-object-cache 2024-09-21T09:42:02.709 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:00:53 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:02.847 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:02.848 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:02.848 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repo Size 2024-09-21T09:42:02.848 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:02.848 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:02.848 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 142 k 2024-09-21T09:42:02.848 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:02.848 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:02.848 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:02.849 INFO:teuthology.orchestra.run.smithi045.stdout:Install 1 Package 2024-09-21T09:42:02.849 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:02.849 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 142 k 2024-09-21T09:42:02.849 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 395 k 2024-09-21T09:42:02.849 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:02.888 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:42:02.889 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:02.889 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:02.889 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:02.889 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:02.889 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:03.033 INFO:teuthology.orchestra.run.smithi045.stdout:ceph-immutable-object-cache-18.2.4-871.g6a1bdf8 775 kB/s | 142 kB 00:00 2024-09-21T09:42:03.034 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:03.034 INFO:teuthology.orchestra.run.smithi045.stdout:Total 768 kB/s | 142 kB 00:00 2024-09-21T09:42:03.034 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:03.042 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:03.042 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:03.102 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:03.103 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:03.107 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-mgr 2024-09-21T09:42:03.262 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:03.326 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:42:03.352 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:42:03.353 INFO:teuthology.orchestra.run.smithi045.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:42:03.353 INFO:teuthology.orchestra.run.smithi045.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:42:03.353 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:03.720 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:55 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:03.812 INFO:teuthology.orchestra.run.smithi032.stdout:Package ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:03.856 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:03.858 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-09-21T09:42:03.858 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:03.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-mgr-dashboard 2024-09-21T09:42:04.335 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:42:04.335 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:04.335 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:04.336 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:04.336 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:04.336 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:04.532 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:56 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:04.579 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-mgr 2024-09-21T09:42:04.670 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-dashboard noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 3.5 M 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-grafana-dashboards noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 24 k 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-prometheus-alerts noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 16 k 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:04.671 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:04.672 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:04.672 INFO:teuthology.orchestra.run.smithi032.stdout:Install 5 Packages 2024-09-21T09:42:04.672 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:04.672 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 3.8 M 2024-09-21T09:42:04.672 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 87 M 2024-09-21T09:42:04.672 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:05.189 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:00:56 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:05.197 INFO:teuthology.orchestra.run.smithi032.stdout:(1/5): ceph-grafana-dashboards-18.2.4-871.g6a1b 159 kB/s | 24 kB 00:00 2024-09-21T09:42:05.222 INFO:teuthology.orchestra.run.smithi032.stdout:(2/5): ceph-prometheus-alerts-18.2.4-871.g6a1bd 92 kB/s | 16 kB 00:00 2024-09-21T09:42:05.264 INFO:teuthology.orchestra.run.smithi032.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 460 kB/s | 31 kB 00:00 2024-09-21T09:42:05.282 INFO:teuthology.orchestra.run.smithi045.stdout:Package ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:05.290 INFO:teuthology.orchestra.run.smithi032.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.8 MB/s | 188 kB 00:00 2024-09-21T09:42:05.327 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:05.329 INFO:teuthology.orchestra.run.smithi045.stdout:Nothing to do. 2024-09-21T09:42:05.329 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:05.416 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-mgr-dashboard 2024-09-21T09:42:05.465 INFO:teuthology.orchestra.run.smithi032.stdout:(5/5): ceph-mgr-dashboard-18.2.4-871.g6a1bdf85. 8.4 MB/s | 3.5 MB 00:00 2024-09-21T09:42:05.468 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:05.469 INFO:teuthology.orchestra.run.smithi032.stdout:Total 4.7 MB/s | 3.8 MB 00:00 2024-09-21T09:42:05.469 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:05.477 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:05.477 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:05.563 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:05.563 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:05.830 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:05.907 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-21T09:42:05.937 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-21T09:42:05.966 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.no 3/5 2024-09-21T09:42:06.023 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:00:57 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:06.160 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:06.161 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:06.161 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2024-09-21T09:42:06.161 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:06.161 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:06.161 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-dashboard noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 3.5 M 2024-09-21T09:42:06.161 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:42:06.161 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-grafana-dashboards noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 24 k 2024-09-21T09:42:06.162 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-prometheus-alerts noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 16 k 2024-09-21T09:42:06.162 INFO:teuthology.orchestra.run.smithi045.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:42:06.162 INFO:teuthology.orchestra.run.smithi045.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:42:06.162 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:06.162 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:06.162 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:06.162 INFO:teuthology.orchestra.run.smithi045.stdout:Install 5 Packages 2024-09-21T09:42:06.162 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:06.162 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 3.8 M 2024-09-21T09:42:06.163 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 87 M 2024-09-21T09:42:06.163 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:06.464 INFO:teuthology.orchestra.run.smithi045.stdout:(1/5): ceph-grafana-dashboards-18.2.4-871.g6a1b 177 kB/s | 24 kB 00:00 2024-09-21T09:42:06.489 INFO:teuthology.orchestra.run.smithi045.stdout:(2/5): ceph-prometheus-alerts-18.2.4-871.g6a1bd 101 kB/s | 16 kB 00:00 2024-09-21T09:42:06.531 INFO:teuthology.orchestra.run.smithi045.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 459 kB/s | 31 kB 00:00 2024-09-21T09:42:06.556 INFO:teuthology.orchestra.run.smithi045.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.8 MB/s | 188 kB 00:00 2024-09-21T09:42:06.723 INFO:teuthology.orchestra.run.smithi045.stdout:(5/5): ceph-mgr-dashboard-18.2.4-871.g6a1bdf85. 8.9 MB/s | 3.5 MB 00:00 2024-09-21T09:42:06.727 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:06.727 INFO:teuthology.orchestra.run.smithi045.stdout:Total 6.7 MB/s | 3.8 MB 00:00 2024-09-21T09:42:06.727 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:06.735 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:06.735 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:06.824 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:06.825 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:07.079 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:07.084 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.n 4/5 2024-09-21T09:42:07.127 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 5/5 2024-09-21T09:42:07.156 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-21T09:42:07.187 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-21T09:42:07.216 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.no 3/5 2024-09-21T09:42:08.246 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 5/5 2024-09-21T09:42:08.246 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.n 1/5 2024-09-21T09:42:08.246 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 2/5 2024-09-21T09:42:08.246 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.no 3/5 2024-09-21T09:42:08.246 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-21T09:42:08.330 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.n 4/5 2024-09-21T09:42:08.370 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 5/5 2024-09-21T09:42:08.502 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-21T09:42:08.502 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:08.502 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:08.502 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:08.502 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:08.502 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:08.502 INFO:teuthology.orchestra.run.smithi032.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:42:08.502 INFO:teuthology.orchestra.run.smithi032.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:42:08.503 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:08.503 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:08.719 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-21T09:42:09.326 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:01 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:09.443 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 5/5 2024-09-21T09:42:09.443 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.n 1/5 2024-09-21T09:42:09.443 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 2/5 2024-09-21T09:42:09.443 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.no 3/5 2024-09-21T09:42:09.443 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-21T09:42:09.465 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:09.466 INFO:teuthology.orchestra.run.smithi032.stdout:====================================================================================== 2024-09-21T09:42:09.466 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-09-21T09:42:09.466 INFO:teuthology.orchestra.run.smithi032.stdout:====================================================================================== 2024-09-21T09:42:09.466 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:09.466 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 7.4 M 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:09.467 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:09.468 INFO:teuthology.orchestra.run.smithi032.stdout:====================================================================================== 2024-09-21T09:42:09.468 INFO:teuthology.orchestra.run.smithi032.stdout:Install 12 Packages 2024-09-21T09:42:09.468 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:09.468 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 38 M 2024-09-21T09:42:09.468 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 212 M 2024-09-21T09:42:09.468 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:09.718 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-21T09:42:09.718 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:09.718 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:09.718 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:09.718 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:09.719 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:09.719 INFO:teuthology.orchestra.run.smithi045.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:42:09.719 INFO:teuthology.orchestra.run.smithi045.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:42:09.719 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:09.719 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:09.942 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-21T09:42:10.022 INFO:teuthology.orchestra.run.smithi032.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.1 MB/s | 191 kB 00:00 2024-09-21T09:42:10.064 INFO:teuthology.orchestra.run.smithi032.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 3.6 MB/s | 801 kB 00:00 2024-09-21T09:42:10.298 INFO:teuthology.orchestra.run.smithi032.stdout:(3/12): ceph-mgr-diskprediction-local-18.2.4-87 16 MB/s | 7.4 MB 00:00 2024-09-21T09:42:10.323 INFO:teuthology.orchestra.run.smithi032.stdout:(4/12): flexiblas-3.0.4-8.el9.x86_64.rpm 109 kB/s | 33 kB 00:00 2024-09-21T09:42:10.524 INFO:teuthology.orchestra.run.smithi032.stdout:(5/12): openblas-0.3.26-2.el9.x86_64.rpm 192 kB/s | 38 kB 00:00 2024-09-21T09:42:10.557 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:01 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:10.582 INFO:teuthology.orchestra.run.smithi032.stdout:(6/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 64 kB/s | 18 kB 00:00 2024-09-21T09:42:10.696 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:10.697 INFO:teuthology.orchestra.run.smithi045.stdout:====================================================================================== 2024-09-21T09:42:10.697 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2024-09-21T09:42:10.697 INFO:teuthology.orchestra.run.smithi045.stdout:====================================================================================== 2024-09-21T09:42:10.697 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:10.697 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 7.4 M 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:42:10.698 INFO:teuthology.orchestra.run.smithi045.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:42:10.699 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:10.699 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:10.699 INFO:teuthology.orchestra.run.smithi045.stdout:====================================================================================== 2024-09-21T09:42:10.699 INFO:teuthology.orchestra.run.smithi045.stdout:Install 12 Packages 2024-09-21T09:42:10.699 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:10.699 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 38 M 2024-09-21T09:42:10.699 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 212 M 2024-09-21T09:42:10.699 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:11.057 INFO:teuthology.orchestra.run.smithi032.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 515 kB/s | 245 kB 00:00 2024-09-21T09:42:11.073 INFO:teuthology.orchestra.run.smithi045.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 921 kB/s | 191 kB 00:00 2024-09-21T09:42:11.132 INFO:teuthology.orchestra.run.smithi045.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 562 kB/s | 33 kB 00:00 2024-09-21T09:42:11.191 INFO:teuthology.orchestra.run.smithi045.stdout:(3/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.4 MB/s | 801 kB 00:00 2024-09-21T09:42:11.333 INFO:teuthology.orchestra.run.smithi032.stdout:(8/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 6.1 MB/s | 4.9 MB 00:00 2024-09-21T09:42:11.333 INFO:teuthology.orchestra.run.smithi045.stdout:(4/12): ceph-mgr-diskprediction-local-18.2.4-87 16 MB/s | 7.4 MB 00:00 2024-09-21T09:42:11.350 INFO:teuthology.orchestra.run.smithi045.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 115 kB/s | 18 kB 00:00 2024-09-21T09:42:11.383 INFO:teuthology.orchestra.run.smithi045.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 770 kB/s | 38 kB 00:00 2024-09-21T09:42:11.449 INFO:teuthology.orchestra.run.smithi032.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.1 MB/s | 250 kB 00:00 2024-09-21T09:42:11.475 INFO:teuthology.orchestra.run.smithi045.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 2.6 MB/s | 245 kB 00:00 2024-09-21T09:42:11.650 INFO:teuthology.orchestra.run.smithi032.stdout:(10/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rp 1.9 MB/s | 3.0 MB 00:01 2024-09-21T09:42:11.709 INFO:teuthology.orchestra.run.smithi045.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 5.2 MB/s | 3.0 MB 00:00 2024-09-21T09:42:11.817 INFO:teuthology.orchestra.run.smithi045.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.3 MB/s | 250 kB 00:00 2024-09-21T09:42:11.908 INFO:teuthology.orchestra.run.smithi032.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 6.0 MB/s | 5.1 MB 00:00 2024-09-21T09:42:12.626 INFO:teuthology.orchestra.run.smithi045.stdout:(10/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 3.9 MB/s | 4.9 MB 00:01 2024-09-21T09:42:12.701 INFO:teuthology.orchestra.run.smithi045.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 4.2 MB/s | 5.1 MB 00:01 2024-09-21T09:42:13.050 INFO:teuthology.orchestra.run.smithi032.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 10 MB/s | 16 MB 00:01 2024-09-21T09:42:13.051 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:13.051 INFO:teuthology.orchestra.run.smithi032.stdout:Total 11 MB/s | 38 MB 00:03 2024-09-21T09:42:13.419 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:13.446 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:13.447 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:13.986 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:13.986 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:13.993 INFO:teuthology.orchestra.run.smithi045.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 7.5 MB/s | 16 MB 00:02 2024-09-21T09:42:13.994 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:13.994 INFO:teuthology.orchestra.run.smithi045.stdout:Total 12 MB/s | 38 MB 00:03 2024-09-21T09:42:14.359 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:14.387 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:14.387 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:14.805 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:14.885 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-21T09:42:14.924 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:42:14.941 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:14.941 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:15.027 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:42:15.067 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-21T09:42:15.257 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-21T09:42:15.290 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-21T09:42:15.377 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-21T09:42:15.732 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:15.800 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-21T09:42:15.850 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:42:15.965 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:42:16.002 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-21T09:42:16.088 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-21T09:42:16.195 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-21T09:42:16.214 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-21T09:42:16.225 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-21T09:42:16.311 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-21T09:42:17.010 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-21T09:42:17.153 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-21T09:42:17.543 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:42:18.502 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:42:18.626 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-21T09:42:18.655 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 12/12 2024-09-21T09:42:19.598 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-21T09:42:19.626 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 12/12 2024-09-21T09:42:21.257 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 12/12 2024-09-21T09:42:21.257 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 1/12 2024-09-21T09:42:21.257 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:42:21.257 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:42:21.257 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-21T09:42:21.258 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-21T09:42:21.258 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-21T09:42:21.258 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-21T09:42:21.258 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-21T09:42:21.258 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-21T09:42:21.258 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:42:21.258 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:42:21.670 INFO:teuthology.orchestra.run.smithi032.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:42:21.671 INFO:teuthology.orchestra.run.smithi032.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:42:21.671 INFO:teuthology.orchestra.run.smithi032.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:42:21.671 INFO:teuthology.orchestra.run.smithi032.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:42:21.671 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:21.671 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:22.052 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-mgr-rook 2024-09-21T09:42:22.307 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 12/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 1/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:42:22.308 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-21T09:42:22.663 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:14 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-21T09:42:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:22.704 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:22.705 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:22.805 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:22.806 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:22.806 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-09-21T09:42:22.806 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:22.806 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:22.806 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-rook noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 49 k 2024-09-21T09:42:22.806 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:42:22.806 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:42:22.806 INFO:teuthology.orchestra.run.smithi032.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:22.807 INFO:teuthology.orchestra.run.smithi032.stdout:Install 13 Packages 2024-09-21T09:42:22.808 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:22.808 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 2.2 M 2024-09-21T09:42:22.808 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 26 M 2024-09-21T09:42:22.808 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:23.013 INFO:teuthology.orchestra.run.smithi032.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 281 kB/s | 26 kB 00:00 2024-09-21T09:42:23.038 INFO:teuthology.orchestra.run.smithi032.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 164 kB/s | 19 kB 00:00 2024-09-21T09:42:23.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-mgr-rook 2024-09-21T09:42:23.096 INFO:teuthology.orchestra.run.smithi032.stdout:(3/13): ceph-mgr-rook-18.2.4-871.g6a1bdf85.el9. 281 kB/s | 49 kB 00:00 2024-09-21T09:42:23.122 INFO:teuthology.orchestra.run.smithi032.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.0 MB/s | 222 kB 00:00 2024-09-21T09:42:23.189 INFO:teuthology.orchestra.run.smithi032.stdout:(5/13): python3-requests-oauthlib-1.3.0-12.el9. 805 kB/s | 54 kB 00:00 2024-09-21T09:42:23.231 INFO:teuthology.orchestra.run.smithi032.stdout:(6/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 823 kB/s | 159 kB 00:00 2024-09-21T09:42:23.265 INFO:teuthology.orchestra.run.smithi032.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 428 kB/s | 32 kB 00:00 2024-09-21T09:42:23.281 INFO:teuthology.orchestra.run.smithi032.stdout:(8/13): python3-certifi-2023.05.07-4.el9.noarch 280 kB/s | 14 kB 00:00 2024-09-21T09:42:23.315 INFO:teuthology.orchestra.run.smithi032.stdout:(9/13): python3-google-auth-2.34.0-1.el9.noarch 4.5 MB/s | 220 kB 00:00 2024-09-21T09:42:23.340 INFO:teuthology.orchestra.run.smithi032.stdout:(10/13): python3-pyasn1-modules-0.4.8-6.el9.noa 1.1 MB/s | 279 kB 00:00 2024-09-21T09:42:23.365 INFO:teuthology.orchestra.run.smithi032.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-09-21T09:42:23.416 INFO:teuthology.orchestra.run.smithi032.stdout:(12/13): python3-kubernetes-26.1.0-2.el9.noarch 7.6 MB/s | 1.0 MB 00:00 2024-09-21T09:42:23.441 INFO:teuthology.orchestra.run.smithi032.stdout:(13/13): python3-websocket-client-1.2.3-2.el9.n 894 kB/s | 90 kB 00:00 2024-09-21T09:42:23.444 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:23.444 INFO:teuthology.orchestra.run.smithi032.stdout:Total 3.5 MB/s | 2.2 MB 00:00 2024-09-21T09:42:23.510 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:23.527 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:23.527 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:23.623 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:23.623 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:23.699 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:14 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:23.840 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:23.841 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:23.841 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2024-09-21T09:42:23.841 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:23.841 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:23.841 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-rook noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 49 k 2024-09-21T09:42:23.841 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:23.842 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:23.843 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:23.843 INFO:teuthology.orchestra.run.smithi045.stdout:Install 13 Packages 2024-09-21T09:42:23.843 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:23.843 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 2.2 M 2024-09-21T09:42:23.843 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 26 M 2024-09-21T09:42:23.843 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:24.142 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:24.313 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-21T09:42:24.369 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-21T09:42:24.413 INFO:teuthology.orchestra.run.smithi045.stdout:(1/13): ceph-mgr-rook-18.2.4-871.g6a1bdf85.el9. 292 kB/s | 49 kB 00:00 2024-09-21T09:42:24.424 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-21T09:42:24.460 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-21T09:42:24.502 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-21T09:42:24.616 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-21T09:42:24.726 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-21T09:42:24.772 INFO:teuthology.orchestra.run.smithi045.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 36 kB/s | 19 kB 00:00 2024-09-21T09:42:24.779 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-21T09:42:24.847 INFO:teuthology.orchestra.run.smithi045.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 42 kB/s | 26 kB 00:00 2024-09-21T09:42:25.155 INFO:teuthology.orchestra.run.smithi045.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 299 kB/s | 222 kB 00:00 2024-09-21T09:42:25.541 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-21T09:42:25.547 INFO:teuthology.orchestra.run.smithi045.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 205 kB/s | 159 kB 00:00 2024-09-21T09:42:25.589 INFO:teuthology.orchestra.run.smithi045.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 783 kB/s | 32 kB 00:00 2024-09-21T09:42:25.615 INFO:teuthology.orchestra.run.smithi045.stdout:(7/13): python3-certifi-2023.05.07-4.el9.noarch 565 kB/s | 14 kB 00:00 2024-09-21T09:42:25.617 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-21T09:42:25.640 INFO:teuthology.orchestra.run.smithi045.stdout:(8/13): python3-google-auth-2.34.0-1.el9.noarch 8.7 MB/s | 220 kB 00:00 2024-09-21T09:42:25.658 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-21T09:42:25.665 INFO:teuthology.orchestra.run.smithi045.stdout:(9/13): python3-pyasn1-modules-0.4.8-6.el9.noar 341 kB/s | 279 kB 00:00 2024-09-21T09:42:25.707 INFO:teuthology.orchestra.run.smithi045.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 1.4 MB/s | 59 kB 00:00 2024-09-21T09:42:25.708 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-21T09:42:25.736 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 13/13 2024-09-21T09:42:25.749 INFO:teuthology.orchestra.run.smithi045.stdout:(11/13): python3-kubernetes-26.1.0-2.el9.noarch 9.4 MB/s | 1.0 MB 00:00 2024-09-21T09:42:25.774 INFO:teuthology.orchestra.run.smithi045.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.3 MB/s | 90 kB 00:00 2024-09-21T09:42:25.800 INFO:teuthology.orchestra.run.smithi045.stdout:(13/13): python3-requests-oauthlib-1.3.0-12.el9 83 kB/s | 54 kB 00:00 2024-09-21T09:42:25.803 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:25.804 INFO:teuthology.orchestra.run.smithi045.stdout:Total 1.1 MB/s | 2.2 MB 00:01 2024-09-21T09:42:25.876 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:25.894 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:25.895 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:25.991 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:25.991 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:26.535 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 13/13 2024-09-21T09:42:26.536 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 1/13 2024-09-21T09:42:26.536 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-21T09:42:26.536 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-21T09:42:26.536 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-21T09:42:26.536 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-21T09:42:26.536 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-21T09:42:26.537 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-21T09:42:26.537 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-21T09:42:26.537 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-21T09:42:26.537 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-21T09:42:26.537 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-21T09:42:26.537 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-21T09:42:26.538 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:26.719 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-21T09:42:26.770 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-21T09:42:26.827 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:42:26.859 INFO:teuthology.orchestra.run.smithi032.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:42:26.860 INFO:teuthology.orchestra.run.smithi032.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:42:26.860 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:42:26.860 INFO:teuthology.orchestra.run.smithi032.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:42:26.860 INFO:teuthology.orchestra.run.smithi032.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:42:26.860 INFO:teuthology.orchestra.run.smithi032.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:42:26.860 INFO:teuthology.orchestra.run.smithi032.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:42:26.860 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:26.860 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:26.861 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-21T09:42:26.904 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-21T09:42:27.011 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-21T09:42:27.116 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-mgr-cephadm 2024-09-21T09:42:27.122 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-21T09:42:27.182 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-21T09:42:27.721 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:19 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:27.860 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:27.860 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:27.860 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-09-21T09:42:27.860 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:27.860 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:27.860 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-cephadm noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 139 k 2024-09-21T09:42:27.860 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:42:27.860 INFO:teuthology.orchestra.run.smithi032.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:42:27.860 INFO:teuthology.orchestra.run.smithi032.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:42:27.861 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:42:27.861 INFO:teuthology.orchestra.run.smithi032.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:42:27.861 INFO:teuthology.orchestra.run.smithi032.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:42:27.861 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:27.861 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:27.861 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:27.861 INFO:teuthology.orchestra.run.smithi032.stdout:Install 6 Packages 2024-09-21T09:42:27.861 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:27.862 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 7.0 M 2024-09-21T09:42:27.862 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 33 M 2024-09-21T09:42:27.862 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:27.965 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-21T09:42:28.011 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-21T09:42:28.045 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-21T09:42:28.094 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-21T09:42:28.122 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 13/13 2024-09-21T09:42:28.501 INFO:teuthology.orchestra.run.smithi032.stdout:(1/6): ceph-mgr-cephadm-18.2.4-871.g6a1bdf85.el 655 kB/s | 139 kB 00:00 2024-09-21T09:42:28.551 INFO:teuthology.orchestra.run.smithi032.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 11 MB/s | 548 kB 00:00 2024-09-21T09:42:28.577 INFO:teuthology.orchestra.run.smithi032.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 2.3 MB/s | 58 kB 00:00 2024-09-21T09:42:28.602 INFO:teuthology.orchestra.run.smithi032.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 2.9 MB/s | 74 kB 00:00 2024-09-21T09:42:28.919 INFO:teuthology.orchestra.run.smithi032.stdout:(5/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 394 kB/s | 248 kB 00:00 2024-09-21T09:42:28.940 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 13/13 2024-09-21T09:42:28.940 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 1/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-21T09:42:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:42:29.276 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:42:29.277 INFO:teuthology.orchestra.run.smithi045.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:42:29.277 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:29.277 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:29.527 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-mgr-cephadm 2024-09-21T09:42:29.969 INFO:teuthology.orchestra.run.smithi032.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.5 MB/s | 6.0 MB 00:01 2024-09-21T09:42:29.971 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:29.971 INFO:teuthology.orchestra.run.smithi032.stdout:Total 3.3 MB/s | 7.0 MB 00:02 2024-09-21T09:42:30.046 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:30.059 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:30.059 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:30.135 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:21 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:30.165 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:30.166 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:30.273 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:30.273 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:30.274 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2024-09-21T09:42:30.274 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:30.274 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:30.274 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-cephadm noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 139 k 2024-09-21T09:42:30.274 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:42:30.274 INFO:teuthology.orchestra.run.smithi045.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:42:30.275 INFO:teuthology.orchestra.run.smithi045.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:42:30.275 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:42:30.275 INFO:teuthology.orchestra.run.smithi045.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:42:30.275 INFO:teuthology.orchestra.run.smithi045.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:42:30.275 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:30.275 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:30.275 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:30.275 INFO:teuthology.orchestra.run.smithi045.stdout:Install 6 Packages 2024-09-21T09:42:30.275 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:30.276 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 7.0 M 2024-09-21T09:42:30.276 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 33 M 2024-09-21T09:42:30.276 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:30.483 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:30.647 INFO:teuthology.orchestra.run.smithi045.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.3 MB/s | 248 kB 00:00 2024-09-21T09:42:30.657 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-21T09:42:30.672 INFO:teuthology.orchestra.run.smithi045.stdout:(2/6): ceph-mgr-cephadm-18.2.4-871.g6a1bdf85.el 674 kB/s | 139 kB 00:00 2024-09-21T09:42:30.713 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-21T09:42:30.722 INFO:teuthology.orchestra.run.smithi045.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 1.1 MB/s | 58 kB 00:00 2024-09-21T09:42:30.747 INFO:teuthology.orchestra.run.smithi045.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 5.3 MB/s | 548 kB 00:00 2024-09-21T09:42:30.773 INFO:teuthology.orchestra.run.smithi045.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.4 MB/s | 74 kB 00:00 2024-09-21T09:42:30.873 INFO:teuthology.orchestra.run.smithi045.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 15 MB/s | 6.0 MB 00:00 2024-09-21T09:42:30.875 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:30.876 INFO:teuthology.orchestra.run.smithi045.stdout:Total 12 MB/s | 7.0 MB 00:00 2024-09-21T09:42:30.949 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:30.961 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:30.961 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:31.068 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:31.068 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:31.265 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-21T09:42:31.351 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-21T09:42:31.383 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:31.420 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-21T09:42:31.449 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 6/6 2024-09-21T09:42:31.562 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-21T09:42:31.617 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-21T09:42:32.179 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-21T09:42:32.239 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 6/6 2024-09-21T09:42:32.239 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/6 2024-09-21T09:42:32.239 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-21T09:42:32.239 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-21T09:42:32.239 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-21T09:42:32.239 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-21T09:42:32.273 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-21T09:42:32.342 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-21T09:42:32.371 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 6/6 2024-09-21T09:42:32.508 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-21T09:42:32.508 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:32.508 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:32.508 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:32.508 INFO:teuthology.orchestra.run.smithi032.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:42:32.508 INFO:teuthology.orchestra.run.smithi032.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:42:32.508 INFO:teuthology.orchestra.run.smithi032.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:42:32.509 INFO:teuthology.orchestra.run.smithi032.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:42:32.509 INFO:teuthology.orchestra.run.smithi032.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:42:32.509 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:32.509 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:32.785 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install ceph-fuse 2024-09-21T09:42:33.118 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 6/6 2024-09-21T09:42:33.118 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/6 2024-09-21T09:42:33.118 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-21T09:42:33.118 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-21T09:42:33.118 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-21T09:42:33.118 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-21T09:42:33.390 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-21T09:42:33.390 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:33.390 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:33.391 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:42:33.391 INFO:teuthology.orchestra.run.smithi045.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:42:33.391 INFO:teuthology.orchestra.run.smithi045.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:42:33.391 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:42:33.391 INFO:teuthology.orchestra.run.smithi045.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:42:33.391 INFO:teuthology.orchestra.run.smithi045.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:42:33.391 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:33.391 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:33.395 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:25 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:33.534 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:33.535 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:33.535 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repository Size 2024-09-21T09:42:33.535 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 853 k 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout:Install 2 Packages 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 932 k 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 2.7 M 2024-09-21T09:42:33.536 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:33.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-fuse 2024-09-21T09:42:33.654 INFO:teuthology.orchestra.run.smithi032.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 847 kB/s | 80 kB 00:00 2024-09-21T09:42:33.854 INFO:teuthology.orchestra.run.smithi032.stdout:(2/2): ceph-fuse-18.2.4-871.g6a1bdf85.el9.x86_6 2.8 MB/s | 853 kB 00:00 2024-09-21T09:42:33.855 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:33.855 INFO:teuthology.orchestra.run.smithi032.stdout:Total 2.9 MB/s | 932 kB 00:00 2024-09-21T09:42:33.867 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:33.890 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:33.890 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:33.975 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:33.975 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:34.176 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:34.259 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:25 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:34.337 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-21T09:42:34.398 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:34.399 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:34.399 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2024-09-21T09:42:34.399 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:34.399 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:34.399 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 853 k 2024-09-21T09:42:34.399 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-21T09:42:34.399 INFO:teuthology.orchestra.run.smithi045.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:42:34.399 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:34.399 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:34.400 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:34.400 INFO:teuthology.orchestra.run.smithi045.stdout:Install 2 Packages 2024-09-21T09:42:34.400 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:34.400 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 932 k 2024-09-21T09:42:34.400 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 2.7 M 2024-09-21T09:42:34.400 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:34.408 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:42:34.559 INFO:teuthology.orchestra.run.smithi045.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 583 kB/s | 80 kB 00:00 2024-09-21T09:42:34.701 INFO:teuthology.orchestra.run.smithi045.stdout:(2/2): ceph-fuse-18.2.4-871.g6a1bdf85.el9.x86_6 3.0 MB/s | 853 kB 00:00 2024-09-21T09:42:34.702 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:34.702 INFO:teuthology.orchestra.run.smithi045.stdout:Total 3.0 MB/s | 932 kB 00:00 2024-09-21T09:42:34.714 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:34.735 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:34.736 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:34.820 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:34.820 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:35.018 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:35.132 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:42:35.132 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/2 2024-09-21T09:42:35.160 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-21T09:42:35.237 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:42:35.435 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-21T09:42:35.435 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:35.435 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:35.435 INFO:teuthology.orchestra.run.smithi032.stdout: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-21T09:42:35.435 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:35.435 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:35.719 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install librados-devel 2024-09-21T09:42:35.898 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:42:35.898 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/2 2024-09-21T09:42:36.182 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-21T09:42:36.182 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:36.182 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:36.182 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-21T09:42:36.182 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:36.182 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:36.332 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:28 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:36.460 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install librados-devel 2024-09-21T09:42:36.468 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:36.469 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:36.469 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repo Size 2024-09-21T09:42:36.469 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:36.469 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:36.469 INFO:teuthology.orchestra.run.smithi032.stdout: librados-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 126 k 2024-09-21T09:42:36.469 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:36.469 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:36.470 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:36.470 INFO:teuthology.orchestra.run.smithi032.stdout:Install 1 Package 2024-09-21T09:42:36.470 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:36.470 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 126 k 2024-09-21T09:42:36.470 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 456 k 2024-09-21T09:42:36.470 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:36.684 INFO:teuthology.orchestra.run.smithi032.stdout:librados-devel-18.2.4-871.g6a1bdf85.el9.x86_64. 594 kB/s | 126 kB 00:00 2024-09-21T09:42:36.684 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:36.685 INFO:teuthology.orchestra.run.smithi032.stdout:Total 589 kB/s | 126 kB 00:00 2024-09-21T09:42:36.685 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:36.691 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:36.691 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:36.758 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:36.758 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:36.914 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:37.042 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:37.065 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:28 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:37.202 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:37.203 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:37.203 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repo Size 2024-09-21T09:42:37.203 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:37.203 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:37.203 INFO:teuthology.orchestra.run.smithi045.stdout: librados-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 126 k 2024-09-21T09:42:37.203 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:37.204 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:37.204 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:37.204 INFO:teuthology.orchestra.run.smithi045.stdout:Install 1 Package 2024-09-21T09:42:37.204 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:37.204 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 126 k 2024-09-21T09:42:37.204 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 456 k 2024-09-21T09:42:37.204 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:37.399 INFO:teuthology.orchestra.run.smithi045.stdout:librados-devel-18.2.4-871.g6a1bdf85.el9.x86_64. 649 kB/s | 126 kB 00:00 2024-09-21T09:42:37.399 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:37.400 INFO:teuthology.orchestra.run.smithi045.stdout:Total 644 kB/s | 126 kB 00:00 2024-09-21T09:42:37.400 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:37.406 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:37.406 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:37.470 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:37.473 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:37.474 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:37.631 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:37.726 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:37.726 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:37.727 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:37.727 INFO:teuthology.orchestra.run.smithi032.stdout: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:37.727 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:37.727 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:37.732 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:38.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install libcephfs2 2024-09-21T09:42:38.189 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:38.450 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:38.450 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:38.450 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:38.451 INFO:teuthology.orchestra.run.smithi045.stdout: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:38.451 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:38.451 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:38.622 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:30 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:38.718 INFO:teuthology.orchestra.run.smithi032.stdout:Package libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:38.728 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install libcephfs2 2024-09-21T09:42:38.764 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:38.765 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-09-21T09:42:38.765 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:38.835 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install libcephfs-devel 2024-09-21T09:42:39.341 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:30 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:39.435 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:31 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:39.436 INFO:teuthology.orchestra.run.smithi045.stdout:Package libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:39.483 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:39.484 INFO:teuthology.orchestra.run.smithi045.stdout:Nothing to do. 2024-09-21T09:42:39.484 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:39.556 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install libcephfs-devel 2024-09-21T09:42:39.572 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:39.573 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:39.573 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repo Size 2024-09-21T09:42:39.573 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:39.573 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:39.573 INFO:teuthology.orchestra.run.smithi032.stdout: libcephfs-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 31 k 2024-09-21T09:42:39.573 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:39.573 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:39.573 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:39.574 INFO:teuthology.orchestra.run.smithi032.stdout:Install 1 Package 2024-09-21T09:42:39.574 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:39.574 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 31 k 2024-09-21T09:42:39.574 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 139 k 2024-09-21T09:42:39.574 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:39.728 INFO:teuthology.orchestra.run.smithi032.stdout:libcephfs-devel-18.2.4-871.g6a1bdf85.el9.x86_64 201 kB/s | 31 kB 00:00 2024-09-21T09:42:39.728 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:39.729 INFO:teuthology.orchestra.run.smithi032.stdout:Total 200 kB/s | 31 kB 00:00 2024-09-21T09:42:39.729 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:39.732 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:39.733 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:39.748 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:39.748 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:39.842 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:39.961 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:40.155 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:31 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:40.292 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repo Size 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout: libcephfs-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 31 k 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout:Install 1 Package 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 31 k 2024-09-21T09:42:40.293 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 139 k 2024-09-21T09:42:40.294 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:40.437 INFO:teuthology.orchestra.run.smithi045.stdout:libcephfs-devel-18.2.4-871.g6a1bdf85.el9.x86_64 216 kB/s | 31 kB 00:00 2024-09-21T09:42:40.438 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:40.438 INFO:teuthology.orchestra.run.smithi045.stdout:Total 214 kB/s | 31 kB 00:00 2024-09-21T09:42:40.438 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:40.442 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:40.442 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:40.462 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:40.463 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:40.559 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:40.609 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:40.609 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:40.609 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:40.609 INFO:teuthology.orchestra.run.smithi032.stdout: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:40.609 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:40.609 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:40.661 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:40.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install librados2 2024-09-21T09:42:41.052 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:41.288 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:41.288 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:41.438 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:33 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:41.484 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install librados2 2024-09-21T09:42:41.530 INFO:teuthology.orchestra.run.smithi032.stdout:Package librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:41.575 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:41.577 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-09-21T09:42:41.577 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:41.667 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install librbd1 2024-09-21T09:42:42.091 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:33 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:42.184 INFO:teuthology.orchestra.run.smithi045.stdout:Package librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:42.229 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:42.230 INFO:teuthology.orchestra.run.smithi045.stdout:Nothing to do. 2024-09-21T09:42:42.230 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:42.273 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:34 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:42.307 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install librbd1 2024-09-21T09:42:42.365 INFO:teuthology.orchestra.run.smithi032.stdout:Package librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:42.411 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:42.412 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-09-21T09:42:42.412 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:42.481 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install python3-rados 2024-09-21T09:42:42.914 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:33 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:43.005 INFO:teuthology.orchestra.run.smithi045.stdout:Package librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:43.051 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:43.052 INFO:teuthology.orchestra.run.smithi045.stdout:Nothing to do. 2024-09-21T09:42:43.052 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:43.084 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:35 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:43.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install python3-rados 2024-09-21T09:42:43.177 INFO:teuthology.orchestra.run.smithi032.stdout:Package python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:43.224 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:43.225 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-09-21T09:42:43.225 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:43.297 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install python3-rgw 2024-09-21T09:42:43.748 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:34 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:43.843 INFO:teuthology.orchestra.run.smithi045.stdout:Package python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:43.889 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:43.890 INFO:teuthology.orchestra.run.smithi045.stdout:Nothing to do. 2024-09-21T09:42:43.891 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:43.904 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:35 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:43.967 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install python3-rgw 2024-09-21T09:42:44.000 INFO:teuthology.orchestra.run.smithi032.stdout:Package python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:44.047 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:44.048 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-09-21T09:42:44.048 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:44.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install python3-cephfs 2024-09-21T09:42:44.575 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:35 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:44.670 INFO:teuthology.orchestra.run.smithi045.stdout:Package python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:44.717 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:44.718 INFO:teuthology.orchestra.run.smithi045.stdout:Nothing to do. 2024-09-21T09:42:44.719 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:44.730 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:36 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:44.798 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install python3-cephfs 2024-09-21T09:42:44.823 INFO:teuthology.orchestra.run.smithi032.stdout:Package python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:44.870 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:44.871 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-09-21T09:42:44.871 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:44.941 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install python3-rbd 2024-09-21T09:42:45.404 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:36 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:45.497 INFO:teuthology.orchestra.run.smithi045.stdout:Package python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:45.543 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:45.544 INFO:teuthology.orchestra.run.smithi045.stdout:Nothing to do. 2024-09-21T09:42:45.544 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:45.548 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:37 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:45.626 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install python3-rbd 2024-09-21T09:42:45.641 INFO:teuthology.orchestra.run.smithi032.stdout:Package python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:45.687 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-09-21T09:42:45.688 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:45.758 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install rbd-fuse 2024-09-21T09:42:46.232 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:37 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:46.326 INFO:teuthology.orchestra.run.smithi045.stdout:Package python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:42:46.366 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:38 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:46.372 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:46.374 INFO:teuthology.orchestra.run.smithi045.stdout:Nothing to do. 2024-09-21T09:42:46.374 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:46.446 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install rbd-fuse 2024-09-21T09:42:46.506 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:46.506 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:46.506 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-09-21T09:42:46.506 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:46.506 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:46.507 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 87 k 2024-09-21T09:42:46.507 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:46.507 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:46.507 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:46.507 INFO:teuthology.orchestra.run.smithi032.stdout:Install 1 Package 2024-09-21T09:42:46.507 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:46.507 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 87 k 2024-09-21T09:42:46.507 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 227 k 2024-09-21T09:42:46.508 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:46.673 INFO:teuthology.orchestra.run.smithi032.stdout:rbd-fuse-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 529 kB/s | 87 kB 00:00 2024-09-21T09:42:46.673 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:46.674 INFO:teuthology.orchestra.run.smithi032.stdout:Total 525 kB/s | 87 kB 00:00 2024-09-21T09:42:46.674 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:46.681 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:46.681 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:46.748 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:46.748 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:47.007 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:47.053 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:38 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:47.191 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:47.191 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:47.191 INFO:teuthology.orchestra.run.smithi045.stdout: Package Architecture Version Repository Size 2024-09-21T09:42:47.191 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 87 k 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout:Install 1 Package 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 87 k 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 227 k 2024-09-21T09:42:47.192 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:47.391 INFO:teuthology.orchestra.run.smithi045.stdout:rbd-fuse-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 440 kB/s | 87 kB 00:00 2024-09-21T09:42:47.391 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:47.392 INFO:teuthology.orchestra.run.smithi045.stdout:Total 437 kB/s | 87 kB 00:00 2024-09-21T09:42:47.392 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:47.399 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:47.399 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:47.416 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:47.466 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:47.467 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:47.617 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:47.674 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:47.674 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:47.674 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:47.674 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:47.674 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:47.674 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:47.724 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:47.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install rbd-mirror 2024-09-21T09:42:48.146 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:48.399 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:48.399 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:48.399 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:48.399 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:48.399 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:48.399 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:48.517 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:40 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:48.628 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install rbd-mirror 2024-09-21T09:42:48.655 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout: Package Arch Version Repo Size 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-mirror x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout:Install 1 Package 2024-09-21T09:42:48.656 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:48.657 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 3.0 M 2024-09-21T09:42:48.657 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 12 M 2024-09-21T09:42:48.657 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:48.984 INFO:teuthology.orchestra.run.smithi032.stdout:rbd-mirror-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 9.2 MB/s | 3.0 MB 00:00 2024-09-21T09:42:48.985 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:48.985 INFO:teuthology.orchestra.run.smithi032.stdout:Total 9.1 MB/s | 3.0 MB 00:00 2024-09-21T09:42:48.985 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:48.995 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:48.995 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:49.079 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:49.079 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:49.240 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:40 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:49.384 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:49.384 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:49.384 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repo Size 2024-09-21T09:42:49.384 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:49.384 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:49.384 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-mirror x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:42:49.385 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:49.385 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:49.385 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:49.385 INFO:teuthology.orchestra.run.smithi045.stdout:Install 1 Package 2024-09-21T09:42:49.385 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:49.385 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 3.0 M 2024-09-21T09:42:49.385 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 12 M 2024-09-21T09:42:49.385 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:49.521 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:49.593 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:49.619 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:49.619 INFO:teuthology.orchestra.run.smithi032.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:42:49.619 INFO:teuthology.orchestra.run.smithi032.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:42:49.619 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:42:49.619 INFO:teuthology.orchestra.run.smithi032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:42:49.619 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:49.712 INFO:teuthology.orchestra.run.smithi045.stdout:rbd-mirror-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 9.2 MB/s | 3.0 MB 00:00 2024-09-21T09:42:49.713 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:49.713 INFO:teuthology.orchestra.run.smithi045.stdout:Total 9.1 MB/s | 3.0 MB 00:00 2024-09-21T09:42:49.713 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:49.723 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:49.723 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:49.808 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:49.808 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:50.246 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:50.305 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:50.331 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:50.331 INFO:teuthology.orchestra.run.smithi045.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:42:50.331 INFO:teuthology.orchestra.run.smithi045.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:42:50.331 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:42:50.331 INFO:teuthology.orchestra.run.smithi045.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:42:50.331 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:50.724 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:50.725 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:50.725 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:50.725 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:50.725 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:50.725 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:50.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum -y install rbd-nbd 2024-09-21T09:42:51.429 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:51.429 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:51.429 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:51.429 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:51.429 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:51.429 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:51.604 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:01:43 ago on Sat 21 Sep 2024 09:41:08 AM UTC. 2024-09-21T09:42:51.681 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install rbd-nbd 2024-09-21T09:42:51.744 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-09-21T09:42:51.745 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:51.745 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-09-21T09:42:51.745 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-nbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 172 k 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout:Install 1 Package 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 172 k 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 494 k 2024-09-21T09:42:51.746 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-09-21T09:42:51.954 INFO:teuthology.orchestra.run.smithi032.stdout:rbd-nbd-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 830 kB/s | 172 kB 00:00 2024-09-21T09:42:51.954 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:51.955 INFO:teuthology.orchestra.run.smithi032.stdout:Total 823 kB/s | 172 kB 00:00 2024-09-21T09:42:51.955 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-09-21T09:42:51.963 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-09-21T09:42:51.963 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-09-21T09:42:52.033 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-09-21T09:42:52.033 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-09-21T09:42:52.200 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-09-21T09:42:52.292 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:01:43 ago on Sat 21 Sep 2024 09:41:09 AM UTC. 2024-09-21T09:42:52.312 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:52.431 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout: Package Architecture Version Repository Size 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-nbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 172 k 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout:Install 1 Package 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 172 k 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 494 k 2024-09-21T09:42:52.432 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-21T09:42:52.651 INFO:teuthology.orchestra.run.smithi045.stdout:rbd-nbd-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 789 kB/s | 172 kB 00:00 2024-09-21T09:42:52.651 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:52.652 INFO:teuthology.orchestra.run.smithi045.stdout:Total 783 kB/s | 172 kB 00:00 2024-09-21T09:42:52.652 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-21T09:42:52.660 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-21T09:42:52.660 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-21T09:42:52.722 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:52.728 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-21T09:42:52.728 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-21T09:42:52.894 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-21T09:42:52.991 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:52.991 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:52.991 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-09-21T09:42:52.991 INFO:teuthology.orchestra.run.smithi032.stdout: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:52.991 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:42:52.991 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-09-21T09:42:53.010 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:53.228 DEBUG:teuthology.parallel:result is None 2024-09-21T09:42:53.478 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:53.730 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:42:53.730 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:53.730 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-21T09:42:53.730 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:42:53.731 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:42:53.731 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-21T09:42:54.008 DEBUG:teuthology.parallel:result is None 2024-09-21T09:42:54.009 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:42:54.009 INFO:teuthology.packaging:ref: None 2024-09-21T09:42:54.009 INFO:teuthology.packaging:tag: None 2024-09-21T09:42:54.009 INFO:teuthology.packaging:branch: reef 2024-09-21T09:42:54.009 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:54.009 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:42:54.264 DEBUG:teuthology.orchestra.run.smithi032:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:42:54.288 INFO:teuthology.orchestra.run.smithi032.stdout:18.2.4-871.g6a1bdf85.el9 2024-09-21T09:42:54.288 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-871.g6a1bdf85.el9 2024-09-21T09:42:54.288 INFO:teuthology.task.install:The correct ceph version 18.2.4-871.g6a1bdf85 is installed. 2024-09-21T09:42:54.290 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:42:54.290 INFO:teuthology.packaging:ref: None 2024-09-21T09:42:54.290 INFO:teuthology.packaging:tag: None 2024-09-21T09:42:54.290 INFO:teuthology.packaging:branch: reef 2024-09-21T09:42:54.291 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:54.291 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:42:54.538 DEBUG:teuthology.orchestra.run.smithi045:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:42:54.559 INFO:teuthology.orchestra.run.smithi045.stdout:18.2.4-871.g6a1bdf85.el9 2024-09-21T09:42:54.560 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-871.g6a1bdf85.el9 2024-09-21T09:42:54.560 INFO:teuthology.task.install:The correct ceph version 18.2.4-871.g6a1bdf85 is installed. 2024-09-21T09:42:54.561 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-21T09:42:54.561 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:42:54.562 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:42:54.590 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:42:54.590 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:42:54.627 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-21T09:42:54.627 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:42:54.627 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:42:54.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:42:54.723 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:42:54.724 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:42:54.750 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:42:54.816 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-21T09:42:54.816 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:42:54.816 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:42:54.843 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:42:54.910 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:42:54.910 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:42:54.935 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:42:55.000 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-21T09:42:55.000 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:42:55.000 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:42:55.026 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:42:55.093 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:42:55.093 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:42:55.118 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:42:55.184 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-21T09:42:55.274 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:42:55.274 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-09-21T09:42:55.274 INFO:tasks.cephadm:Cluster fsid is e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:42:55.274 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-21T09:42:55.275 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.32', 'mon.c': '[v2:172.21.15.32:3301,v1:172.21.15.32:6790]', 'mon.b': '172.21.15.45'} 2024-09-21T09:42:55.275 INFO:tasks.cephadm:First mon is mon.a on smithi032 2024-09-21T09:42:55.275 INFO:tasks.cephadm:First mgr is y 2024-09-21T09:42:55.275 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-21T09:42:55.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo hostname $(hostname -s) 2024-09-21T09:42:55.302 DEBUG:teuthology.orchestra.run.smithi045:> sudo hostname $(hostname -s) 2024-09-21T09:42:55.328 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-09-21T09:42:55.328 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:55.469 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/56129/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.3.0-5092-g62da54b4', 'node_name': '172.21.2.5+braggi05', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.3.0-5092.g62da54b4'}, 'url': 'https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-09-20 21:56:15.050681', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'main', 'chacra_url': 'https://3.chacra.ceph.com/repos/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/', 'archs': ['source', 'x86_64'], 'distro': 'centos'}] 2024-09-21T09:42:55.751 INFO:tasks.util.chacra:got chacra host 4.chacra.ceph.com, ref reef, sha1 6a1bdf854bb5803efd58957907442aa01f41a8fc from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&ref=reef 2024-09-21T09:42:55.754 INFO:tasks.cephadm:Discovered cachra url: https://4.chacra.ceph.com/binaries/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/x86_64/flavors/default/cephadm 2024-09-21T09:42:55.754 INFO:tasks.cephadm:Downloading cephadm from url: https://4.chacra.ceph.com/binaries/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/x86_64/flavors/default/cephadm 2024-09-21T09:42:55.755 DEBUG:teuthology.orchestra.run.smithi032:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:42:56.231 INFO:teuthology.orchestra.run.smithi032.stdout:-rw-r--r--. 1 ubuntu ubuntu 217222 Sep 21 09:42 /home/ubuntu/cephtest/cephadm 2024-09-21T09:42:56.231 DEBUG:teuthology.orchestra.run.smithi045:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:42:56.698 INFO:teuthology.orchestra.run.smithi045.stdout:-rw-r--r--. 1 ubuntu ubuntu 217222 Sep 21 09:42 /home/ubuntu/cephtest/cephadm 2024-09-21T09:42:56.698 DEBUG:teuthology.orchestra.run.smithi032:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:42:56.718 DEBUG:teuthology.orchestra.run.smithi045:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:42:56.749 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-09-21T09:42:56.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-21T09:42:56.760 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-21T09:42:58.335 INFO:teuthology.orchestra.run.smithi045.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-21T09:42:58.388 INFO:teuthology.orchestra.run.smithi032.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-21T09:43:27.181 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T09:43:27.181 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_version": "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)", 2024-09-21T09:43:27.181 INFO:teuthology.orchestra.run.smithi045.stdout: "image_id": "0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de", 2024-09-21T09:43:27.181 INFO:teuthology.orchestra.run.smithi045.stdout: "repo_digests": [ 2024-09-21T09:43:27.181 INFO:teuthology.orchestra.run.smithi045.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6" 2024-09-21T09:43:27.181 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T09:43:27.181 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T09:43:28.317 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:43:28.318 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph_version": "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)", 2024-09-21T09:43:28.318 INFO:teuthology.orchestra.run.smithi032.stdout: "image_id": "0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de", 2024-09-21T09:43:28.318 INFO:teuthology.orchestra.run.smithi032.stdout: "repo_digests": [ 2024-09-21T09:43:28.318 INFO:teuthology.orchestra.run.smithi032.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6" 2024-09-21T09:43:28.318 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T09:43:28.318 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:43:28.338 DEBUG:teuthology.orchestra.run.smithi032:> sudo mkdir -p /etc/ceph 2024-09-21T09:43:28.382 DEBUG:teuthology.orchestra.run.smithi045:> sudo mkdir -p /etc/ceph 2024-09-21T09:43:28.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 777 /etc/ceph 2024-09-21T09:43:28.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod 777 /etc/ceph 2024-09-21T09:43:28.485 INFO:tasks.cephadm:Writing seed config... 2024-09-21T09:43:28.486 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-21T09:43:28.486 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-21T09:43:28.486 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-21T09:43:28.486 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-21T09:43:28.486 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-21T09:43:28.486 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-09-21T09:43:28.487 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-21T09:43:28.487 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-21T09:43:28.487 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-21T09:43:28.487 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-21T09:43:28.487 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-21T09:43:28.487 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:43:28.487 DEBUG:teuthology.orchestra.run.smithi032:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-21T09:43:28.503 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = e0bde4d4-77fd-11ef-baf6-efdc52797490 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-21T09:43:28.504 DEBUG:teuthology.orchestra.run.smithi032:mon.a> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service 2024-09-21T09:43:28.547 DEBUG:teuthology.orchestra.run.smithi032:mgr.y> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y.service 2024-09-21T09:43:28.590 INFO:tasks.cephadm:Bootstrapping... 2024-09-21T09:43:28.590 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.32 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:43:28.748 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:43:28.748 INFO:teuthology.orchestra.run.smithi032.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', 'e0bde4d4-77fd-11ef-baf6-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.32', '--skip-admin-label'] 2024-09-21T09:43:28.782 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:43:28.783 INFO:teuthology.orchestra.run.smithi032.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-21T09:43:28.783 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying podman|docker is present... 2024-09-21T09:43:28.805 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:43:28.805 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying lvm2 is present... 2024-09-21T09:43:28.805 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying time synchronization is in place... 2024-09-21T09:43:28.813 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:43:28.813 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:43:28.820 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:43:28.820 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-09-21T09:43:28.827 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout enabled 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout active 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi032.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi032.stdout:Repeating the final host check... 2024-09-21T09:43:28.855 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:43:28.856 INFO:teuthology.orchestra.run.smithi032.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-21T09:43:28.856 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl is present 2024-09-21T09:43:28.856 INFO:teuthology.orchestra.run.smithi032.stdout:lvcreate is present 2024-09-21T09:43:28.862 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:43:28.863 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:43:28.869 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:43:28.869 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-09-21T09:43:28.875 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout enabled 2024-09-21T09:43:28.882 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout active 2024-09-21T09:43:28.883 INFO:teuthology.orchestra.run.smithi032.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:43:28.883 INFO:teuthology.orchestra.run.smithi032.stdout:Host looks OK 2024-09-21T09:43:28.883 INFO:teuthology.orchestra.run.smithi032.stdout:Cluster fsid: e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:28.883 INFO:teuthology.orchestra.run.smithi032.stdout:Acquiring lock 139906280509008 on /run/cephadm/e0bde4d4-77fd-11ef-baf6-efdc52797490.lock 2024-09-21T09:43:28.883 INFO:teuthology.orchestra.run.smithi032.stdout:Lock 139906280509008 acquired on /run/cephadm/e0bde4d4-77fd-11ef-baf6-efdc52797490.lock 2024-09-21T09:43:28.883 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying IP 172.21.15.32 port 3300 ... 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying IP 172.21.15.32 port 6789 ... 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi032.stdout:Base mon IP(s) is [172.21.15.32:3300, 172.21.15.32:6789], mon addrv is [v2:172.21.15.32:3300,v1:172.21.15.32:6789] 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.32 metric 100 2024-09-21T09:43:28.887 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.32 metric 100 2024-09-21T09:43:28.889 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-21T09:43:28.889 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-21T09:43:28.889 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1796sec hoplimit 64 pref medium 2024-09-21T09:43:28.892 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-21T09:43:28.892 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-21T09:43:28.892 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:43:28.892 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-21T09:43:28.892 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fef8:2602/64 scope link 2024-09-21T09:43:28.892 INFO:teuthology.orchestra.run.smithi032.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:43:28.893 INFO:teuthology.orchestra.run.smithi032.stdout:Mon IP `172.21.15.32` is in CIDR network `172.21.0.0/20` 2024-09-21T09:43:28.893 INFO:teuthology.orchestra.run.smithi032.stdout:Mon IP `172.21.15.32` is in CIDR network `172.21.0.0/20` 2024-09-21T09:43:28.893 INFO:teuthology.orchestra.run.smithi032.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-21T09:43:28.893 INFO:teuthology.orchestra.run.smithi032.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-21T09:43:28.893 INFO:teuthology.orchestra.run.smithi032.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-21T09:43:29.379 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stdout 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de 2024-09-21T09:43:29.380 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:reef... 2024-09-21T09:43:29.380 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Getting image source signatures 2024-09-21T09:43:29.380 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Copying blob sha256:b51f6d1e10fe8de9e06c08c0a23ce0010d2bef42d30a6f89f3ac07fd857715ba 2024-09-21T09:43:29.380 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-21T09:43:29.380 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Copying config sha256:0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de 2024-09-21T09:43:29.380 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-21T09:43:30.275 INFO:teuthology.orchestra.run.smithi032.stdout:ceph: stdout ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable) 2024-09-21T09:43:30.275 INFO:teuthology.orchestra.run.smithi032.stdout:Ceph version: ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable) 2024-09-21T09:43:30.275 INFO:teuthology.orchestra.run.smithi032.stdout:Extracting ceph user uid/gid from container image... 2024-09-21T09:43:31.215 INFO:teuthology.orchestra.run.smithi032.stdout:stat: stdout 167 167 2024-09-21T09:43:31.215 INFO:teuthology.orchestra.run.smithi032.stdout:Creating initial keys... 2024-09-21T09:43:32.153 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph-authtool: stdout AQBDle5miS+JIRAA/7K+Gq7guKSgtYJuU4yDSQ== 2024-09-21T09:43:33.217 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph-authtool: stdout AQBEle5m0HyWHRAAYIl3iJ/eI4Fy/xkT5THZHw== 2024-09-21T09:43:34.140 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph-authtool: stdout AQBFle5mx1hQIBAAccm3mzy4VcDohKuL8rBBRg== 2024-09-21T09:43:34.140 INFO:teuthology.orchestra.run.smithi032.stdout:Creating initial monmap... 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout:monmaptool for a [v2:172.21.15.32:3300,v1:172.21.15.32:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout:setting min_mon_release = pacific 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: set fsid to e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:43:34.988 INFO:teuthology.orchestra.run.smithi032.stdout:Creating mon... 2024-09-21T09:43:35.885 INFO:teuthology.orchestra.run.smithi032.stdout:create mon.a on 2024-09-21T09:43:36.108 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-21T09:43:36.286 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-21T09:43:36.489 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490.target. 2024-09-21T09:43:36.489 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490.target. 2024-09-21T09:43:36.694 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a 2024-09-21T09:43:36.694 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Failed to reset failed state of unit ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service: Unit ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service not loaded. 2024-09-21T09:43:36.858 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490.target.wants/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service → /etc/systemd/system/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@.service. 2024-09-21T09:43:37.354 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:37 smithi032 ceph-mon[28820]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:43:37.354 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:37 smithi032 bash[28802]: 1c5aabef837c2ab0842420f920b1bb2bfcf5e8b76174651be8ec39424171bb45 2024-09-21T09:43:37.367 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:43:37.367 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout disabled 2024-09-21T09:43:37.374 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:43:37.374 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-09-21T09:43:37.374 INFO:teuthology.orchestra.run.smithi032.stdout:firewalld.service is not enabled 2024-09-21T09:43:37.374 INFO:teuthology.orchestra.run.smithi032.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:43:37.375 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mon to start... 2024-09-21T09:43:37.375 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mon... 2024-09-21T09:43:37.606 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:37 smithi032 systemd[1]: Started Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:43:38.347 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout cluster: 2024-09-21T09:43:38.347 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout id: e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout services: 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.545853s) 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout data: 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-21T09:43:38.348 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-21T09:43:38.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-21T09:43:38.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout pgs: 2024-09-21T09:43:38.349 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:38.349 INFO:teuthology.orchestra.run.smithi032.stdout:mon is available 2024-09-21T09:43:38.349 INFO:teuthology.orchestra.run.smithi032.stdout:Assimilating anything we can from ceph.conf... 2024-09-21T09:43:38.578 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:38 smithi032 ceph-mon[28820]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:43:38.579 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:38 smithi032 ceph-mon[28820]: monmap e1: 1 mons at {a=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:43:38.579 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:38 smithi032 ceph-mon[28820]: fsmap 2024-09-21T09:43:38.579 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:38 smithi032 ceph-mon[28820]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:43:38.579 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:38 smithi032 ceph-mon[28820]: mgrmap e1: no daemons active 2024-09-21T09:43:38.579 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:38 smithi032 ceph-mon[28820]: from='client.? 172.21.15.32:0/703570594' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T09:43:39.362 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:39.362 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:43:39.362 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout fsid = e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:39.362 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.32:3300,v1:172.21.15.32:6789] 2024-09-21T09:43:39.362 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:43:39.362 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:43:39.362 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:43:39.363 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:43:39.363 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:39.363 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:43:39.363 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:43:39.363 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:39.363 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:43:39.363 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:43:39.363 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:43:39.363 INFO:teuthology.orchestra.run.smithi032.stdout:Generating new minimal ceph.conf... 2024-09-21T09:43:39.570 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:39 smithi032 ceph-mon[28820]: from='client.? 172.21.15.32:0/2013478441' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:43:39.570 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:39 smithi032 ceph-mon[28820]: from='client.? 172.21.15.32:0/2013478441' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-21T09:43:40.367 INFO:teuthology.orchestra.run.smithi032.stdout:Restarting the monitor... 2024-09-21T09:43:40.651 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:40 smithi032 ceph-mon[28820]: from='client.? 172.21.15.32:0/1057026777' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:40.651 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:40 smithi032 systemd[1]: Stopping Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:43:40.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:40 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[28816]: 2024-09-21T09:43:40.649+0000 7fbbb09b9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:43:40.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:40 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[28816]: 2024-09-21T09:43:40.649+0000 7fbbb09b9640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-21T09:43:40.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:40 smithi032 podman[29063]: 2024-09-21 09:43:40.772247329 +0000 UTC m=+0.234613362 container died 1c5aabef837c2ab0842420f920b1bb2bfcf5e8b76174651be8ec39424171bb45 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T09:43:41.292 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 podman[29063]: 2024-09-21 09:43:41.065163168 +0000 UTC m=+0.527529192 container cleanup 1c5aabef837c2ab0842420f920b1bb2bfcf5e8b76174651be8ec39424171bb45 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T09:43:41.292 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 bash[29063]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:43:41.626 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 podman[29075]: 2024-09-21 09:43:41.291072458 +0000 UTC m=+0.512838607 container remove 1c5aabef837c2ab0842420f920b1bb2bfcf5e8b76174651be8ec39424171bb45 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T09:43:41.626 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:43:41.626 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 systemd[1]: Stopped Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:43:41.626 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 systemd[1]: Starting Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:43:41.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 podman[29172]: 2024-09-21 09:43:41.625378912 +0000 UTC m=+0.111464783 container create 017caecda0d912368af69361a2cdc938fe1158bfdb788902e6d707c4e3ce0dbb (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:43:41.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 podman[29172]: 2024-09-21 09:43:41.530375488 +0000 UTC m=+0.016461359 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph:reef 2024-09-21T09:43:41.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 podman[29172]: 2024-09-21 09:43:41.844413571 +0000 UTC m=+0.330499454 container init 017caecda0d912368af69361a2cdc938fe1158bfdb788902e6d707c4e3ce0dbb (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T09:43:41.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 podman[29172]: 2024-09-21 09:43:41.847590161 +0000 UTC m=+0.333676037 container start 017caecda0d912368af69361a2cdc938fe1158bfdb788902e6d707c4e3ce0dbb (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T09:43:41.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:43:41.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable), process ceph-mon, pid 2 2024-09-21T09:43:41.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: pidfile_write: ignore empty --pid-file 2024-09-21T09:43:41.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: load: jerasure load: lrc 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Git sha 0 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Compile date 2024-09-20 20:34:00 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: DB SUMMARY 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: DB Session ID: C2KWFKE573EPDI90EOGJ 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84544 ; 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:43:41.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.env: 0x55eba089dbc0 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.info_log: 0x55eba1e0abe0 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.statistics: (nil) 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.use_fsync: 0 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:43:41.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.db_log_dir: 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.wal_dir: 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:43:41.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:43:41.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.write_buffer_manager: 0x55eba1e18320 2024-09-21T09:43:41.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:43:41.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:43:41.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:43:41.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:43:41.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:43:41.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:43:41.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:43:41.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:43:41.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.unordered_write: 0 2024-09-21T09:43:41.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:43:41.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:43:41.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:43:41.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.row_cache: None 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.wal_filter: None 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.wal_compression: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:43:41.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:43:41.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_open_files: -1 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Compression algorithms supported: 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: kZSTD supported: 0 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:43:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.merge_operator: 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_filter: None 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:43:41.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55eba1e0a7e0) 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: index_type: 0 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: data_block_index_type: 0 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: index_shortening: 1 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: checksum: 4 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: no_block_cache: 0 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache: 0x55eba1dff350 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:43:41.887 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache_options: 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: capacity : 536870912 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: num_shard_bits : 4 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: strict_capacity_limit : 0 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache_compressed: (nil) 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: persistent_cache: (nil) 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_size: 4096 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_size_deviation: 10 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_restart_interval: 16 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: index_block_restart_interval: 1 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: metadata_block_size: 4096 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: partition_filters: 0 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: use_delta_encoding: 1 2024-09-21T09:43:41.888 INFO:journalctl@ceph.mon.a.smithi032.stdout: filter_policy: bloomfilter 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: whole_key_filtering: 1 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: verify_compression: 0 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: format_version: 5 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: enable_index_compression: 1 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_align: 0 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: max_auto_readahead_size: 262144 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: prepopulate_block_cache: 0 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression: NoCompression 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.num_levels: 7 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:43:41.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:43:41.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:43:41.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.ttl: 2592000 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:43:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3a81636a-4603-4766-8cf1-db3baa1b39f3 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911821874517, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911821875478, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81094, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79244, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73445, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911821, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3a81636a-4603-4766-8cf1-db3baa1b39f3", "db_session_id": "C2KWFKE573EPDI90EOGJ", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:43:41.895 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911821875584, "job": 1, "event": "recovery_finished"} 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55eba1e2ae00 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: DB pointer 0x55eba1f1e000 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** DB Stats ** 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:43:41.896 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** Compaction Stats [default] ** 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: L0 2/0 81.05 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: Sum 2/0 81.05 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** Compaction Stats [default] ** 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 124.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:43:41.897 INFO:journalctl@ceph.mon.a.smithi032.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative compaction: 0.00 GB write, 23.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval compaction: 0.00 GB write, 23.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: Block cache BinnedLRUCache@0x55eba1dff350#2 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:43:41.898 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:43:41.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: starting mon.a rank 0 at public addrs [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] at bind addrs [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:41.944 INFO:teuthology.orchestra.run.smithi032.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-21T09:43:42.147 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: mon.a@-1(???) e1 preinit fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: mon.a@-1(???).mds e1 new map 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: mon.a@-1(???).mds e1 print_map 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout: e1 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout: legacy client fscid: -1 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout: No filesystems configured 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 ceph-mon[29191]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-21T09:43:42.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 bash[29172]: 017caecda0d912368af69361a2cdc938fe1158bfdb788902e6d707c4e3ce0dbb 2024-09-21T09:43:42.149 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:41 smithi032 systemd[1]: Started Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:43:42.994 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:42 smithi032 ceph-mon[29191]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:43:42.994 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:42 smithi032 ceph-mon[29191]: monmap e1: 1 mons at {a=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:43:42.994 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:42 smithi032 ceph-mon[29191]: fsmap 2024-09-21T09:43:42.994 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:42 smithi032 ceph-mon[29191]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:43:42.994 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:42 smithi032 ceph-mon[29191]: mgrmap e1: no daemons active 2024-09-21T09:43:42.994 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:42 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/538413503' entity='client.admin' 2024-09-21T09:43:42.997 INFO:teuthology.orchestra.run.smithi032.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-21T09:43:43.010 INFO:teuthology.orchestra.run.smithi032.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:43:43.010 INFO:teuthology.orchestra.run.smithi032.stdout:Creating mgr... 2024-09-21T09:43:43.010 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-21T09:43:43.011 INFO:teuthology.orchestra.run.smithi032.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-21T09:43:43.240 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y 2024-09-21T09:43:43.241 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Failed to reset failed state of unit ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y.service: Unit ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y.service not loaded. 2024-09-21T09:43:43.394 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490.target.wants/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y.service → /etc/systemd/system/ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@.service. 2024-09-21T09:43:43.620 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:43 smithi032 systemd[1]: Starting Ceph mgr.y for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:43:43.923 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:43 smithi032 podman[29402]: 2024-09-21 09:43:43.619092772 +0000 UTC m=+0.108357991 container create c1adf1e5a4653876430bf5157f305dd3547a4aeb9fc84fa5cafe9da754d50933 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-21T09:43:43.923 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:43 smithi032 podman[29402]: 2024-09-21 09:43:43.527183039 +0000 UTC m=+0.016448273 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph:reef 2024-09-21T09:43:43.923 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:43 smithi032 podman[29402]: 2024-09-21 09:43:43.846364217 +0000 UTC m=+0.335629439 container init c1adf1e5a4653876430bf5157f305dd3547a4aeb9fc84fa5cafe9da754d50933 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-21T09:43:43.923 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:43 smithi032 podman[29402]: 2024-09-21 09:43:43.849111806 +0000 UTC m=+0.338377019 container start c1adf1e5a4653876430bf5157f305dd3547a4aeb9fc84fa5cafe9da754d50933 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:43:43.923 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:43 smithi032 bash[29402]: c1adf1e5a4653876430bf5157f305dd3547a4aeb9fc84fa5cafe9da754d50933 2024-09-21T09:43:43.937 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:43:43.937 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout disabled 2024-09-21T09:43:43.944 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:43:43.945 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-09-21T09:43:43.945 INFO:teuthology.orchestra.run.smithi032.stdout:firewalld.service is not enabled 2024-09-21T09:43:43.945 INFO:teuthology.orchestra.run.smithi032.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:43:43.953 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:43:43.954 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout disabled 2024-09-21T09:43:43.961 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:43:43.962 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-09-21T09:43:43.962 INFO:teuthology.orchestra.run.smithi032.stdout:firewalld.service is not enabled 2024-09-21T09:43:43.962 INFO:teuthology.orchestra.run.smithi032.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-21T09:43:43.962 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mgr to start... 2024-09-21T09:43:43.962 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mgr... 2024-09-21T09:43:44.178 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:43 smithi032 systemd[1]: Started Ceph mgr.y for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:43:44.178 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:43 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:43.973+0000 7fd42be3c040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:43:44.178 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:44.036+0000 7fd42be3c040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:43:44.445 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:44.197+0000 7fd42be3c040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:43:44.445 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:44.266+0000 7fd42be3c040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:43:44.445 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:44.443+0000 7fd42be3c040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:43:44.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:44 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3298134650' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:43:44.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:44.597+0000 7fd42be3c040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:43:45.122 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:44.925+0000 7fd42be3c040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:43:45.123 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:44.988+0000 7fd42be3c040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:43:45.123 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:45.126 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:43:45.126 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsid": "e0bde4d4-77fd-11ef-baf6-efdc52797490", 2024-09-21T09:43:45.126 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:45.127 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:43:45.128 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:43:45.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:43:45.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:43:45.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:43:45.129 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:43:45.131 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:43:37.317773+0000", 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:43:45.132 INFO:teuthology.orchestra.run.smithi032.stdout:mgr not available, waiting (1/15)... 2024-09-21T09:43:45.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:45 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:45.179+0000 7fd42be3c040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:43:46.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:46 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:46.089+0000 7fd42be3c040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:43:46.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:46 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:46.157+0000 7fd42be3c040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:43:47.183 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:46 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:46.912+0000 7fd42be3c040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:43:47.183 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.031+0000 7fd42be3c040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:43:47.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.181+0000 7fd42be3c040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:43:47.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.245+0000 7fd42be3c040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:43:47.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.401+0000 7fd42be3c040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:43:47.705 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/278452600' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:43:47.705 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.472+0000 7fd42be3c040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:43:47.705 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.536+0000 7fd42be3c040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:43:47.705 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.703+0000 7fd42be3c040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:43:47.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.771+0000 7fd42be3c040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:43:47.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.843+0000 7fd42be3c040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:43:47.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.930+0000 7fd42be3c040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:43:48.252 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:47.992+0000 7fd42be3c040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:43:48.252 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:48.054+0000 7fd42be3c040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:43:48.257 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:48.257 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:43:48.257 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsid": "e0bde4d4-77fd-11ef-baf6-efdc52797490", 2024-09-21T09:43:48.257 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:43:48.257 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:43:48.257 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:43:48.257 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:43:48.257 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:48.257 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:43:48.258 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:48.259 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:43:48.259 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:43:48.259 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:43:48.259 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:43:48.262 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:43:48.262 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:43:48.262 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:48.262 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:43:48.262 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:43:48.262 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:43:48.262 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:43:48.262 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:43:48.262 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:43:48.263 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:43:37.317773+0000", 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:43:48.264 INFO:teuthology.orchestra.run.smithi032.stdout:mgr not available, waiting (2/15)... 2024-09-21T09:43:49.206 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:48.873+0000 7fd42be3c040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:43:49.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: Activating manager daemon y 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: mgrmap e2: y(active, starting, since 0.00295548s) 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: Manager daemon y is now available 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' 2024-09-21T09:43:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:48 smithi032 ceph-mon[29191]: from='mgr.14100 172.21.15.32:0/1996777390' entity='mgr.y' 2024-09-21T09:43:50.942 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:50 smithi032 ceph-mon[29191]: mgrmap e3: y(active, since 1.00704s) 2024-09-21T09:43:51.466 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsid": "e0bde4d4-77fd-11ef-baf6-efdc52797490", 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:43:51.467 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:43:51.468 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:43:51.469 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:43:37.317773+0000", 2024-09-21T09:43:51.470 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:43:51.471 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:43:51.471 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:43:51.471 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:43:51.471 INFO:teuthology.orchestra.run.smithi032.stdout:mgr is available 2024-09-21T09:43:51.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:51 smithi032 ceph-mon[29191]: mgrmap e4: y(active, since 2s) 2024-09-21T09:43:51.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:51 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3725787872' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:43:52.618 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:52.618 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:43:52.618 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout fsid = e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:52.618 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:43:52.618 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:43:52.618 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:43:52.619 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:43:52.619 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:52.619 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:43:52.619 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:43:52.619 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 2024-09-21T09:43:52.619 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:43:52.619 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:43:52.619 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:43:52.619 INFO:teuthology.orchestra.run.smithi032.stdout:Enabling cephadm module... 2024-09-21T09:43:53.067 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:52 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/4267827736' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:43:54.187 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:53 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1336181708' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-21T09:43:54.187 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:53 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ignoring --setuser ceph since I am not root 2024-09-21T09:43:54.187 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:53 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ignoring --setgroup ceph since I am not root 2024-09-21T09:43:54.188 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:54 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:54.032+0000 7f2c3b525040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:43:54.188 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:54 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:54.090+0000 7f2c3b525040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:43:54.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:54 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:54.246+0000 7f2c3b525040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:43:54.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:54 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:54.317+0000 7f2c3b525040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:43:54.886 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:54 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:54.468+0000 7f2c3b525040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:43:54.886 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:54 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:54.615+0000 7f2c3b525040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:43:55.192 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:54 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1336181708' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-21T09:43:55.192 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:54 smithi032 ceph-mon[29191]: mgrmap e5: y(active, since 5s) 2024-09-21T09:43:55.192 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:54 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:54.936+0000 7f2c3b525040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:43:55.192 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:55 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:55.000+0000 7f2c3b525040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:43:55.193 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:55 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:55.189+0000 7f2c3b525040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:43:56.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:56 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:56.096+0000 7f2c3b525040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:43:56.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:56 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:56.169+0000 7f2c3b525040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:43:57.206 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:56 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:56.929+0000 7f2c3b525040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:43:57.206 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.050+0000 7f2c3b525040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:43:57.476 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.201+0000 7f2c3b525040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:43:57.476 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.270+0000 7f2c3b525040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:43:57.476 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.426+0000 7f2c3b525040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:43:57.732 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.499+0000 7f2c3b525040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:43:57.732 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.562+0000 7f2c3b525040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:43:57.733 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.730+0000 7f2c3b525040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:43:58.021 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.799+0000 7f2c3b525040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:43:58.021 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.868+0000 7f2c3b525040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:43:58.021 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:57 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:57.956+0000 7f2c3b525040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:43:58.021 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:58.018+0000 7f2c3b525040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:43:58.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:58.083+0000 7f2c3b525040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:43:59.206 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:43:58.897+0000 7f2c3b525040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: Active manager daemon y restarted 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: Activating manager daemon y 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: osdmap e2: 0 total, 0 up, 0 in 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: mgrmap e6: y(active, starting, since 0.00325993s) 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: Manager daemon y is now available 2024-09-21T09:43:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: Found migration_current of "None". Setting to last migration. 2024-09-21T09:43:59.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:43:59.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:43:59.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:59.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:59.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:43:59.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:58 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:44:00.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:59 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2889273859' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:44:00.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:59 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:00.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:59 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:43:59 smithi032 ceph-mon[29191]: mgrmap e7: y(active, since 1.00479s) 2024-09-21T09:44:01.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:01.281+0000 7f2ba3205640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:44:01] ENGINE Error in HTTPServer.serve 2024-09-21T09:44:01.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: Traceback (most recent call last): 2024-09-21T09:44:01.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:44:01.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self._connections.run(self.expiration_interval) 2024-09-21T09:44:01.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:44:01.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self._run(expiration_interval) 2024-09-21T09:44:01.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:44:01.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: s = self.context.wrap_socket( 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: return self.sslsocket_class._create( 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self.do_handshake() 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self._sslobj.do_handshake() 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:44:01.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-mon[29191]: mgrmap e8: y(active, since 2s) 2024-09-21T09:44:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:01] ENGINE Bus STARTING 2024-09-21T09:44:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:01] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:44:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:01] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:44:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:01] ENGINE Bus STARTED 2024-09-21T09:44:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:01 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:01] ENGINE Error in HTTPServer.serve 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: self.do_handshake() 2024-09-21T09:44:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:44:02.209 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:44:02.209 INFO:journalctl@ceph.mon.a.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:44:05.547 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:44:05.548 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 6, 2024-09-21T09:44:05.548 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:44:05.548 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:44:05.548 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:44:05.548 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:44:05.548 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for the mgr to restart... 2024-09-21T09:44:05.548 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mgr epoch 6... 2024-09-21T09:44:06.996 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:44:06.996 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 8, 2024-09-21T09:44:06.996 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:44:06.996 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:44:06.996 INFO:teuthology.orchestra.run.smithi032.stdout:mgr epoch 6 is available 2024-09-21T09:44:06.996 INFO:teuthology.orchestra.run.smithi032.stdout:Setting orchestrator backend to cephadm... 2024-09-21T09:44:07.256 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:06 smithi032 ceph-mon[29191]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:44:07.256 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:06 smithi032 ceph-mon[29191]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:44:08.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:08 smithi032 ceph-mon[29191]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:08.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:08 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:08.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:08 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:09.502 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-21T09:44:09.502 INFO:teuthology.orchestra.run.smithi032.stdout:Generating ssh key... 2024-09-21T09:44:09.902 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:09 smithi032 ceph-mon[29191]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: Generating public/private rsa key pair. 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: Your identification has been saved in /tmp/tmpwhl_fg6y/key 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: Your public key has been saved in /tmp/tmpwhl_fg6y/key.pub 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: The key fingerprint is: 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: SHA256:r6KHrYASywX9wQfvuzF7iZJr9G9Oim3gTPGhdrpOkbE ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: The key's randomart image is: 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: +---[RSA 3072]----+ 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: | . | 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: | . . o | 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: | . . + o | 2024-09-21T09:44:10.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: | . ..B. | 2024-09-21T09:44:10.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: |. . E+.S | 2024-09-21T09:44:10.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: |.+. *.o.. | 2024-09-21T09:44:10.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: |+.. *+*+..o | 2024-09-21T09:44:10.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: |. .oB*+B= | 2024-09-21T09:44:10.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: | =O=**o | 2024-09-21T09:44:10.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: +----[SHA256]-----+ 2024-09-21T09:44:11.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:11 smithi032 ceph-mon[29191]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:11 smithi032 ceph-mon[29191]: Generating ssh key... 2024-09-21T09:44:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:11 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:11 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:12.331 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCox3SAnd4qQLhAms2nScsi03nOrN8uJNpT28VtofAhBKJyQJlvpkma3s55HEppVb0cIBJVL38wVmLNl7Tz55KQtEcA57knfrPqyYQNKUhOOPVuT0QBNmJ4ws+0l2lc1xgPNYXvgB1Lbji6B5blDv1vV5wGdHdMEywtXB1rcFRChKxeXWeEi2CZhOR4Er6UVVSyZhQuCRWamu9vzQp1QdsAMCgPEgG64+NQCs8js42//GHEAs3guavTgHuWVyVJ04NTwcQkn3Ov1GHDS1TvOTz6jCxYRJXVuHqs3gqOK+wfjkWprbrfKr7UjJq1XhixMI6+B8YRbAFMsSu40BqVIDpWtaYUXavytBl+ZoV5Ha3/Gortl0s5E0M4PNTOYqi6R/uqF3Hp+2esEf4BL60leONyEY0wiMiPlFrBVAHDJDCV+/MeQ/s83jZ+w2b4bobreumFoILRnBvrryPj1zVPdWKZf0DQC+rmIZr/ysJA/oGCa1X7ZxB3aCXnlKY5hfh9Sks= ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:44:12.332 INFO:teuthology.orchestra.run.smithi032.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-21T09:44:12.332 INFO:teuthology.orchestra.run.smithi032.stdout:Adding key to root@localhost authorized_keys... 2024-09-21T09:44:12.336 INFO:teuthology.orchestra.run.smithi032.stdout:Adding host smithi032... 2024-09-21T09:44:12.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:12 smithi032 ceph-mon[29191]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:13.669 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:13 smithi032 ceph-mon[29191]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi032", "addr": "172.21.15.32", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:15.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:14 smithi032 ceph-mon[29191]: Deploying cephadm binary to smithi032 2024-09-21T09:44:16.932 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout Added host 'smithi032' with addr '172.21.15.32' 2024-09-21T09:44:16.933 INFO:teuthology.orchestra.run.smithi032.stdout:Deploying unmanaged mon service... 2024-09-21T09:44:17.568 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:17 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:17.568 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:17 smithi032 ceph-mon[29191]: Added host smithi032 2024-09-21T09:44:17.569 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:17 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:18.493 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-21T09:44:18.493 INFO:teuthology.orchestra.run.smithi032.stdout:Deploying unmanaged mgr service... 2024-09-21T09:44:19.074 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:18 smithi032 ceph-mon[29191]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:19.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:18 smithi032 ceph-mon[29191]: Saving service mon spec with placement count:5 2024-09-21T09:44:19.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:18 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:20.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:19 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:20.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:19 smithi032 ceph-mon[29191]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:20.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:19 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:20.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:19 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:20.715 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-21T09:44:21.025 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:20 smithi032 ceph-mon[29191]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:21.025 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:20 smithi032 ceph-mon[29191]: Saving service mgr spec with placement count:2 2024-09-21T09:44:21.025 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:20 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:21.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:21 smithi032 ceph-mon[29191]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:21.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:21 smithi032 ceph-mon[29191]: from='mgr.14118 172.21.15.32:0/4115937366' entity='mgr.y' 2024-09-21T09:44:21.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:21 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/476198481' entity='client.admin' 2024-09-21T09:44:23.836 INFO:teuthology.orchestra.run.smithi032.stdout:Enabling the dashboard module... 2024-09-21T09:44:24.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:23 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2041108285' entity='client.admin' 2024-09-21T09:44:24.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:23 smithi032 ceph-mon[29191]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:25.147 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:24 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/557694823' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-21T09:44:25.147 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ignoring --setuser ceph since I am not root 2024-09-21T09:44:25.147 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ignoring --setgroup ceph since I am not root 2024-09-21T09:44:25.147 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:24.937+0000 7f674cc46040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:44:25.148 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:24.996+0000 7f674cc46040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:44:25.431 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:25 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:25.153+0000 7f674cc46040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:44:25.431 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:25 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:25.221+0000 7f674cc46040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:44:25.431 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:25 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:25.372+0000 7f674cc46040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:44:25.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:25 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:25.522+0000 7f674cc46040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:44:26.065 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:25 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/557694823' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-21T09:44:26.065 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:25 smithi032 ceph-mon[29191]: mgrmap e9: y(active, since 25s) 2024-09-21T09:44:26.065 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:25 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:25.850+0000 7f674cc46040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:44:26.065 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:25 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:25.914+0000 7f674cc46040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:44:26.370 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:26 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:26.109+0000 7f674cc46040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:44:26.665 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:44:26.666 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-21T09:44:26.666 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:44:26.666 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:44:26.666 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:44:26.666 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:44:26.666 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for the mgr to restart... 2024-09-21T09:44:26.666 INFO:teuthology.orchestra.run.smithi032.stdout:Waiting for mgr epoch 9... 2024-09-21T09:44:26.916 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:26 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/225443505' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:44:27.206 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:27 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:27.037+0000 7f674cc46040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:44:27.206 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:27 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:27.108+0000 7f674cc46040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:44:28.124 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:27 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:27.856+0000 7f674cc46040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:44:28.124 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:27 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:27.975+0000 7f674cc46040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:44:28.415 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.122+0000 7f674cc46040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:44:28.416 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.186+0000 7f674cc46040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:44:28.416 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.342+0000 7f674cc46040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:44:28.416 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.414+0000 7f674cc46040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:44:28.695 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.478+0000 7f674cc46040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:44:28.695 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.646+0000 7f674cc46040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:44:28.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.715+0000 7f674cc46040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:44:28.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.786+0000 7f674cc46040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:44:28.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.873+0000 7f674cc46040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:44:28.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.937+0000 7f674cc46040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:44:29.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:28 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:28.998+0000 7f674cc46040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:44:30.070 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:29.815+0000 7f674cc46040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: Active manager daemon y restarted 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: Activating manager daemon y 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: osdmap e3: 0 total, 0 up, 0 in 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: mgrmap e10: y(active, starting, since 0.00306654s) 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: Manager daemon y is now available 2024-09-21T09:44:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:30.072 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:44:30.072 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:44:31.573 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:31.452+0000 7f66b30d0640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:44:31] ENGINE Error in HTTPServer.serve 2024-09-21T09:44:31.573 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: Traceback (most recent call last): 2024-09-21T09:44:31.573 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:44:31.573 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self._connections.run(self.expiration_interval) 2024-09-21T09:44:31.573 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:44:31.573 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self._run(expiration_interval) 2024-09-21T09:44:31.573 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: s = self.context.wrap_socket( 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: return self.sslsocket_class._create( 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self.do_handshake() 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self._sslobj.do_handshake() 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:44:31.574 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:44:31.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:44:31.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-21T09:44:31.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:44:31.575 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:44:31.575 INFO:teuthology.orchestra.run.smithi032.stdout:mgr epoch 9 is available 2024-09-21T09:44:31.576 INFO:teuthology.orchestra.run.smithi032.stdout:Generating a dashboard self-signed certificate... 2024-09-21T09:44:31.909 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-mon[29191]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:44:31.909 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-mon[29191]: mgrmap e11: y(active, since 1.00527s) 2024-09-21T09:44:31.909 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:31 smithi032 ceph-mon[29191]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:44:32.816 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-21T09:44:32.816 INFO:teuthology.orchestra.run.smithi032.stdout:Creating initial admin user... 2024-09-21T09:44:33.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:32 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:31] ENGINE Bus STARTING 2024-09-21T09:44:33.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:32 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:31] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:32 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:31] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:32 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:31] ENGINE Bus STARTED 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:32 smithi032 ceph-mon[29191]: [21/Sep/2024:09:44:31] ENGINE Error in HTTPServer.serve 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:44:33.125 INFO:journalctl@ceph.mon.a.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout: self.do_handshake() 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:32 smithi032 ceph-mon[29191]: mgrmap e12: y(active, since 2s) 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:33.126 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:33.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:33 smithi032 ceph-mon[29191]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:33.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:34.331 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$EF08O.OQDxrmeP3cFsme1OIWPY98f2ISXOqwFhHVw1y.7U.P/YmyW", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726911873, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-21T09:44:34.331 INFO:teuthology.orchestra.run.smithi032.stdout:Fetching dashboard port number... 2024-09-21T09:44:34.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:34 smithi032 ceph-mon[29191]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:35.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:35 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2533282087' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-21T09:44:36.370 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stdout 8443 2024-09-21T09:44:36.382 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:44:36.382 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout disabled 2024-09-21T09:44:36.391 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:44:36.391 INFO:teuthology.orchestra.run.smithi032.stdout:systemctl: stdout inactive 2024-09-21T09:44:36.391 INFO:teuthology.orchestra.run.smithi032.stdout:firewalld.service is not enabled 2024-09-21T09:44:36.391 INFO:teuthology.orchestra.run.smithi032.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-21T09:44:36.393 INFO:teuthology.orchestra.run.smithi032.stdout:Ceph Dashboard is now available at: 2024-09-21T09:44:36.393 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:36.393 INFO:teuthology.orchestra.run.smithi032.stdout: URL: https://smithi032.front.sepia.ceph.com:8443/ 2024-09-21T09:44:36.393 INFO:teuthology.orchestra.run.smithi032.stdout: User: admin 2024-09-21T09:44:36.393 INFO:teuthology.orchestra.run.smithi032.stdout: Password: xdxfcgcu0c 2024-09-21T09:44:36.393 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:36.393 INFO:teuthology.orchestra.run.smithi032.stdout:Saving cluster configuration to /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config directory 2024-09-21T09:44:36.393 INFO:teuthology.orchestra.run.smithi032.stdout:Enabling autotune for osd_memory_target 2024-09-21T09:44:36.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:36 smithi032 ceph-mon[29191]: mgrmap e13: y(active, since 6s) 2024-09-21T09:44:38.800 INFO:teuthology.orchestra.run.smithi032.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout:Or, if you are only running a single cluster on this host: 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout: ceph telemetry on 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:38.801 INFO:teuthology.orchestra.run.smithi032.stdout:For more information see: 2024-09-21T09:44:38.802 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:38.802 INFO:teuthology.orchestra.run.smithi032.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-09-21T09:44:38.802 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:38.802 INFO:teuthology.orchestra.run.smithi032.stdout:Bootstrap complete. 2024-09-21T09:44:38.838 INFO:tasks.cephadm:Fetching config... 2024-09-21T09:44:38.838 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:44:38.838 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-21T09:44:38.855 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-21T09:44:38.855 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:44:38.856 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-21T09:44:38.911 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-21T09:44:38.911 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:44:38.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/keyring of=/dev/stdout 2024-09-21T09:44:38.978 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-21T09:44:38.978 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:44:38.978 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-21T09:44:39.035 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-21T09:44:39.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCox3SAnd4qQLhAms2nScsi03nOrN8uJNpT28VtofAhBKJyQJlvpkma3s55HEppVb0cIBJVL38wVmLNl7Tz55KQtEcA57knfrPqyYQNKUhOOPVuT0QBNmJ4ws+0l2lc1xgPNYXvgB1Lbji6B5blDv1vV5wGdHdMEywtXB1rcFRChKxeXWeEi2CZhOR4Er6UVVSyZhQuCRWamu9vzQp1QdsAMCgPEgG64+NQCs8js42//GHEAs3guavTgHuWVyVJ04NTwcQkn3Ov1GHDS1TvOTz6jCxYRJXVuHqs3gqOK+wfjkWprbrfKr7UjJq1XhixMI6+B8YRbAFMsSu40BqVIDpWtaYUXavytBl+ZoV5Ha3/Gortl0s5E0M4PNTOYqi6R/uqF3Hp+2esEf4BL60leONyEY0wiMiPlFrBVAHDJDCV+/MeQ/s83jZ+w2b4bobreumFoILRnBvrryPj1zVPdWKZf0DQC+rmIZr/ysJA/oGCa1X7ZxB3aCXnlKY5hfh9Sks= ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:44:39.115 INFO:teuthology.orchestra.run.smithi032.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCox3SAnd4qQLhAms2nScsi03nOrN8uJNpT28VtofAhBKJyQJlvpkma3s55HEppVb0cIBJVL38wVmLNl7Tz55KQtEcA57knfrPqyYQNKUhOOPVuT0QBNmJ4ws+0l2lc1xgPNYXvgB1Lbji6B5blDv1vV5wGdHdMEywtXB1rcFRChKxeXWeEi2CZhOR4Er6UVVSyZhQuCRWamu9vzQp1QdsAMCgPEgG64+NQCs8js42//GHEAs3guavTgHuWVyVJ04NTwcQkn3Ov1GHDS1TvOTz6jCxYRJXVuHqs3gqOK+wfjkWprbrfKr7UjJq1XhixMI6+B8YRbAFMsSu40BqVIDpWtaYUXavytBl+ZoV5Ha3/Gortl0s5E0M4PNTOYqi6R/uqF3Hp+2esEf4BL60leONyEY0wiMiPlFrBVAHDJDCV+/MeQ/s83jZ+w2b4bobreumFoILRnBvrryPj1zVPdWKZf0DQC+rmIZr/ysJA/oGCa1X7ZxB3aCXnlKY5hfh9Sks= ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:44:39.129 DEBUG:teuthology.orchestra.run.smithi045:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCox3SAnd4qQLhAms2nScsi03nOrN8uJNpT28VtofAhBKJyQJlvpkma3s55HEppVb0cIBJVL38wVmLNl7Tz55KQtEcA57knfrPqyYQNKUhOOPVuT0QBNmJ4ws+0l2lc1xgPNYXvgB1Lbji6B5blDv1vV5wGdHdMEywtXB1rcFRChKxeXWeEi2CZhOR4Er6UVVSyZhQuCRWamu9vzQp1QdsAMCgPEgG64+NQCs8js42//GHEAs3guavTgHuWVyVJ04NTwcQkn3Ov1GHDS1TvOTz6jCxYRJXVuHqs3gqOK+wfjkWprbrfKr7UjJq1XhixMI6+B8YRbAFMsSu40BqVIDpWtaYUXavytBl+ZoV5Ha3/Gortl0s5E0M4PNTOYqi6R/uqF3Hp+2esEf4BL60leONyEY0wiMiPlFrBVAHDJDCV+/MeQ/s83jZ+w2b4bobreumFoILRnBvrryPj1zVPdWKZf0DQC+rmIZr/ysJA/oGCa1X7ZxB3aCXnlKY5hfh9Sks= ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:44:39.176 INFO:teuthology.orchestra.run.smithi045.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCox3SAnd4qQLhAms2nScsi03nOrN8uJNpT28VtofAhBKJyQJlvpkma3s55HEppVb0cIBJVL38wVmLNl7Tz55KQtEcA57knfrPqyYQNKUhOOPVuT0QBNmJ4ws+0l2lc1xgPNYXvgB1Lbji6B5blDv1vV5wGdHdMEywtXB1rcFRChKxeXWeEi2CZhOR4Er6UVVSyZhQuCRWamu9vzQp1QdsAMCgPEgG64+NQCs8js42//GHEAs3guavTgHuWVyVJ04NTwcQkn3Ov1GHDS1TvOTz6jCxYRJXVuHqs3gqOK+wfjkWprbrfKr7UjJq1XhixMI6+B8YRbAFMsSu40BqVIDpWtaYUXavytBl+ZoV5Ha3/Gortl0s5E0M4PNTOYqi6R/uqF3Hp+2esEf4BL60leONyEY0wiMiPlFrBVAHDJDCV+/MeQ/s83jZ+w2b4bobreumFoILRnBvrryPj1zVPdWKZf0DQC+rmIZr/ysJA/oGCa1X7ZxB3aCXnlKY5hfh9Sks= ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:44:39.189 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-21T09:44:39.393 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:39.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:39 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1554478005' entity='client.admin' 2024-09-21T09:44:42.628 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-21T09:44:42.628 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-21T09:44:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:42.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:42 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/4204935094' entity='client.admin' 2024-09-21T09:44:44.669 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:44 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:44.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:44 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:44.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:44 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi032", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:44.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:44 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:44.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:44 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:44.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:44 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:45.799 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi045 2024-09-21T09:44:45.799 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:44:45.799 DEBUG:teuthology.orchestra.run.smithi045:> dd of=/etc/ceph/ceph.conf 2024-09-21T09:44:45.819 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:44:45.819 DEBUG:teuthology.orchestra.run.smithi045:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:44:45.878 INFO:tasks.cephadm:Adding host smithi045 to orchestrator... 2024-09-21T09:44:45.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch host add smithi045 2024-09-21T09:44:46.088 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:46.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:46.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:46.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:46.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:46.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:46.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:44:46.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:44:46.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:44:46.356 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:44:46.356 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:46.356 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:46.356 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:48.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:47 smithi032 ceph-mon[29191]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi045", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:50.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:49 smithi032 ceph-mon[29191]: Deploying cephadm binary to smithi045 2024-09-21T09:44:50.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:49 smithi032 ceph-mon[29191]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:51.161 INFO:teuthology.orchestra.run.smithi032.stdout:Added host 'smithi045' with addr '172.21.15.45' 2024-09-21T09:44:51.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch host ls --format=json 2024-09-21T09:44:51.967 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:52.330 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:52 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:52.330 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:52 smithi032 ceph-mon[29191]: Added host smithi045 2024-09-21T09:44:52.330 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:52 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:52.330 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:52 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:52.330 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:52 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:52.330 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:52 smithi032 ceph-mon[29191]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:53.588 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:44:53.588 INFO:teuthology.orchestra.run.smithi032.stdout:[{"addr": "172.21.15.32", "hostname": "smithi032", "labels": [], "status": ""}, {"addr": "172.21.15.45", "hostname": "smithi045", "labels": [], "status": ""}] 2024-09-21T09:44:53.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:54.194 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-21T09:44:54.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd crush tunables default 2024-09-21T09:44:54.394 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:54.782 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:54 smithi032 ceph-mon[29191]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:44:54.782 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:54 smithi032 ceph-mon[29191]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi045", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:44:56.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:44:56.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:44:56.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:44:56.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:56.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:56.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:56.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:44:56.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:56 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1567799986' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T09:44:57.039 INFO:teuthology.orchestra.run.smithi032.stderr:adjusted tunables profile to default 2024-09-21T09:44:57.629 INFO:tasks.cephadm:Adding mon.a on smithi032 2024-09-21T09:44:57.629 INFO:tasks.cephadm:Adding mon.c on smithi032 2024-09-21T09:44:57.629 INFO:tasks.cephadm:Adding mon.b on smithi045 2024-09-21T09:44:57.629 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch apply mon '3;smithi032:172.21.15.32=a;smithi032:[v2:172.21.15.32:3301,v1:172.21.15.32:6790]=c;smithi045:172.21.15.45=b' 2024-09-21T09:44:57.835 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:44:58.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:58 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1567799986' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:44:58.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:58 smithi032 ceph-mon[29191]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:44:58.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:44:58 smithi032 ceph-mon[29191]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:59.495 INFO:teuthology.orchestra.run.smithi045.stdout:Scheduled mon update... 2024-09-21T09:45:00.111 DEBUG:teuthology.orchestra.run.smithi032:mon.c> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.c.service 2024-09-21T09:45:00.114 DEBUG:teuthology.orchestra.run.smithi045:mon.b> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.b.service 2024-09-21T09:45:00.116 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:45:00.116 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:45:00.817 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi032:172.21.15.32=a;smithi032:[v2:172.21.15.32:3301,v1:172.21.15.32:6790]=c;smithi045:172.21.15.45=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: Saving service mon spec with placement smithi032:172.21.15.32=a;smithi032:[v2:172.21.15.32:3301,v1:172.21.15.32:6790]=c;smithi045:172.21.15.45=b;count:3 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: Deploying daemon mon.b on smithi045 2024-09-21T09:45:00.899 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:00 smithi032 ceph-mon[29191]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:02.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:01 smithi032 ceph-mon[29191]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:03.942 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:45:03.942 INFO:teuthology.orchestra.run.smithi045.stdout:{"epoch":1,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","modified":"2024-09-21T09:43:34.453975Z","created":"2024-09-21T09:43:34.453975Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T09:45:03.942 INFO:teuthology.orchestra.run.smithi045.stderr:dumped monmap epoch 1 2024-09-21T09:45:04.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:03 smithi032 ceph-mon[29191]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:04.905 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:04 smithi045 ceph-mon[33087]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-21T09:45:05.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:04 smithi045 bash[33068]: 071e4635dabcea783b4a4a5389b6195dfc96fdc4977316509d545bdf36ced58b 2024-09-21T09:45:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:04 smithi045 systemd[1]: Started Ceph mon.b for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:45:05.491 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:45:05.491 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:45:05.712 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:45:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: mon.a calling monitor election 2024-09-21T09:45:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: mon.b calling monitor election 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: monmap e2: 2 mons at {a=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: fsmap 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: mgrmap e13: y(active, since 40s) 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: overall HEALTH_OK 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:10.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:10.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:10.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: mon.a calling monitor election 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: mon.b calling monitor election 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: monmap e2: 2 mons at {a=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: fsmap 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: mgrmap e13: y(active, since 40s) 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: overall HEALTH_OK 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:10.487 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:45:10.487 INFO:teuthology.orchestra.run.smithi045.stdout:{"epoch":2,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","modified":"2024-09-21T09:45:04.885207Z","created":"2024-09-21T09:43:34.453975Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-21T09:45:10.488 INFO:teuthology.orchestra.run.smithi045.stderr:dumped monmap epoch 2 2024-09-21T09:45:11.069 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:10 smithi045 ceph-mon[33087]: Deploying daemon mon.c on smithi032 2024-09-21T09:45:11.069 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:10 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/3637867147' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:45:11.069 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:10 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:11.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:10 smithi032 ceph-mon[29191]: Deploying daemon mon.c on smithi032 2024-09-21T09:45:11.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:10 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/3637867147' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:45:11.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:10 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:12.093 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:45:12.093 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:45:12.198 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:11 smithi032 ceph-mon[29191]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:12.198 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:45:11 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:45:11.885+0000 7f66ec315640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:45:12.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:11 smithi045 ceph-mon[33087]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:12.305 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Git sha 0 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Compile date 2024-09-20 20:34:00 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: DB SUMMARY 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: DB Session ID: XPSBD9PMFQS4APOU812G 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:45:13.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.env: 0x558da2031bc0 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.info_log: 0x558da3a3cbe0 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.statistics: (nil) 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.use_fsync: 0 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:45:13.111 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.db_log_dir: 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.wal_dir: 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:45:13.113 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:45:13.113 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:45:13.113 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.write_buffer_manager: 0x558da3a4a320 2024-09-21T09:45:13.113 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:45:13.113 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:45:13.113 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:45:13.113 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:45:13.113 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:45:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:45:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:45:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:45:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.unordered_write: 0 2024-09-21T09:45:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:45:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:45:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:45:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:45:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.row_cache: None 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.wal_filter: None 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.wal_compression: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:45:13.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:45:13.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_open_files: -1 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Compression algorithms supported: 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: kZSTD supported: 0 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:45:13.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.merge_operator: 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_filter: None 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:45:13.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558da3a3c7e0) 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: index_type: 0 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: data_block_index_type: 0 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: index_shortening: 1 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: checksum: 4 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: no_block_cache: 0 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_cache: 0x558da3a31350 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_cache_options: 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: capacity : 536870912 2024-09-21T09:45:13.462 INFO:journalctl@ceph.mon.c.smithi032.stdout: num_shard_bits : 4 2024-09-21T09:45:13.463 INFO:journalctl@ceph.mon.c.smithi032.stdout: strict_capacity_limit : 0 2024-09-21T09:45:13.463 INFO:journalctl@ceph.mon.c.smithi032.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:45:13.463 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_cache_compressed: (nil) 2024-09-21T09:45:13.463 INFO:journalctl@ceph.mon.c.smithi032.stdout: persistent_cache: (nil) 2024-09-21T09:45:13.463 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_size: 4096 2024-09-21T09:45:13.463 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_size_deviation: 10 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_restart_interval: 16 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: index_block_restart_interval: 1 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: metadata_block_size: 4096 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: partition_filters: 0 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: use_delta_encoding: 1 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: filter_policy: bloomfilter 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: whole_key_filtering: 1 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: verify_compression: 0 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:45:13.464 INFO:journalctl@ceph.mon.c.smithi032.stdout: format_version: 5 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout: enable_index_compression: 1 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_align: 0 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout: max_auto_readahead_size: 262144 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout: prepopulate_block_cache: 0 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression: NoCompression 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.num_levels: 7 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:45:13.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:45:13.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:45:13.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:45:13.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:45:13.469 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.ttl: 2592000 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:45:13.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 10357bed-1921-4fe3-96ca-02d127d4b05a 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911913108801, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911913109263, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1773, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911913, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "10357bed-1921-4fe3-96ca-02d127d4b05a", "db_session_id": "XPSBD9PMFQS4APOU812G", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911913109375, "job": 1, "event": "recovery_finished"} 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x558da3a5ce00 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: DB pointer 0x558da3b50000 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout: ** DB Stats ** 2024-09-21T09:45:13.471 INFO:journalctl@ceph.mon.c.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: ** Compaction Stats [default] ** 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: L0 1/0 1.73 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Sum 1/0 1.73 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: ** Compaction Stats [default] ** 2024-09-21T09:45:13.472 INFO:journalctl@ceph.mon.c.smithi032.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: Cumulative compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: Interval compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: Block cache BinnedLRUCache@0x558da3a31350#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-09-21T09:45:13.473 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: using public_addrv [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: starting mon.c rank -1 at public addrs [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] at bind addrs [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(???) e0 preinit fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).mds e1 new map 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout: e1 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout: legacy client fscid: -1 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout: No filesystems configured 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-21T09:45:13.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi032:172.21.15.32=a;smithi032:[v2:172.21.15.32:3301,v1:172.21.15.32:6790]=c;smithi045:172.21.15.45=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: Saving service mon spec with placement smithi032:172.21.15.32=a;smithi032:[v2:172.21.15.32:3301,v1:172.21.15.32:6790]=c;smithi045:172.21.15.45=b;count:3 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:13.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: Deploying daemon mon.b on smithi045 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.a calling monitor election 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.b calling monitor election 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:13.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: monmap e2: 2 mons at {a=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: fsmap 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mgrmap e13: y(active, since 40s) 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: overall HEALTH_OK 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:13.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: Deploying daemon mon.c on smithi032 2024-09-21T09:45:13.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/3637867147' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:45:13.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:13.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:13.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 ceph-mon[40061]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-21T09:45:13.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 bash[40035]: 125675da4b8420a5ce1f33b4822cb4ddcd4062f3126d03e4427027f306416813 2024-09-21T09:45:13.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:13 smithi032 systemd[1]: Started Ceph mon.c for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:45:18.417 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:45:18.417 INFO:teuthology.orchestra.run.smithi045.stdout:{"epoch":3,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","modified":"2024-09-21T09:45:13.133999Z","created":"2024-09-21T09:43:34.453975Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3301","nonce":0},{"type":"v1","addr":"172.21.15.32:6790","nonce":0}]},"addr":"172.21.15.32:6790/0","public_addr":"172.21.15.32:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-21T09:45:18.418 INFO:teuthology.orchestra.run.smithi045.stderr:dumped monmap epoch 3 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: mon.a calling monitor election 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: mon.b calling monitor election 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: mon.c calling monitor election 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: monmap e3: 3 mons at {a=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],c=[v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: fsmap 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: mgrmap e13: y(active, since 48s) 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: overall HEALTH_OK 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: mon.a calling monitor election 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: mon.b calling monitor election 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: mon.c calling monitor election 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: monmap e3: 3 mons at {a=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],c=[v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: fsmap 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: mgrmap e13: y(active, since 48s) 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: overall HEALTH_OK 2024-09-21T09:45:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: mon.a calling monitor election 2024-09-21T09:45:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: mon.b calling monitor election 2024-09-21T09:45:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: mon.c calling monitor election 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: monmap e3: 3 mons at {a=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],c=[v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: fsmap 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: mgrmap e13: y(active, since 48s) 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: overall HEALTH_OK 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:18.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:19.019 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-21T09:45:19.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph config generate-minimal-conf 2024-09-21T09:45:19.227 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:45:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:19 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:19 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/4178674155' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:45:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:19 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:19.347 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:19 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:19.348 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:19 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/4178674155' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:45:19.348 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:19 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:19.348 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:19 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:19.348 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:19 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/4178674155' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:45:19.348 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:19 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:20.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:45:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:20.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:20.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:45:21.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:45:21.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:45:21.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[29191]: Reconfiguring daemon mon.a on smithi032 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:45:21.391 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:21.392 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:45:21.392 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:21.392 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:21 smithi032 ceph-mon[40061]: Reconfiguring daemon mon.a on smithi032 2024-09-21T09:45:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:45:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:45:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:45:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:21.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:45:21.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:21.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:21 smithi045 ceph-mon[33087]: Reconfiguring daemon mon.a on smithi032 2024-09-21T09:45:21.653 INFO:teuthology.orchestra.run.smithi032.stdout:# minimal ceph.conf for e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:45:21.653 INFO:teuthology.orchestra.run.smithi032.stdout:[global] 2024-09-21T09:45:21.653 INFO:teuthology.orchestra.run.smithi032.stdout: fsid = e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:45:21.653 INFO:teuthology.orchestra.run.smithi032.stdout: mon_host = [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/4244709768' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[29191]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/4244709768' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[40061]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:22.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:22.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:22.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:45:22.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:22 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:22.494 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-21T09:45:22.494 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:45:22.494 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:45:22.523 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:45:22.523 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:45:22.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:22 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/4244709768' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:22.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:22 smithi045 ceph-mon[33087]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:22.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:22 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:22 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:22 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:22 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:45:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:22 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:22.589 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:45:22.589 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:45:22.622 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:45:22.623 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:45:22.692 INFO:tasks.cephadm:Adding mgr.y on smithi032 2024-09-21T09:45:22.692 INFO:tasks.cephadm:Adding mgr.x on smithi045 2024-09-21T09:45:22.693 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch apply mgr '2;smithi032=y;smithi045=x' 2024-09-21T09:45:22.935 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:45:23.367 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:23 smithi045 ceph-mon[33087]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:45:23.368 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:23 smithi045 ceph-mon[33087]: Reconfiguring daemon mon.c on smithi032 2024-09-21T09:45:23.655 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:23 smithi032 ceph-mon[29191]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:45:23.655 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:23 smithi032 ceph-mon[29191]: Reconfiguring daemon mon.c on smithi032 2024-09-21T09:45:23.656 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:23 smithi032 ceph-mon[40061]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:45:23.656 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:23 smithi032 ceph-mon[40061]: Reconfiguring daemon mon.c on smithi032 2024-09-21T09:45:25.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:24 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:24 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:24 smithi045 ceph-mon[33087]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:45:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:24 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:24 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:45:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:24 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:24 smithi045 ceph-mon[33087]: Reconfiguring daemon mon.b on smithi045 2024-09-21T09:45:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:24 smithi045 ceph-mon[33087]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[40061]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:45:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:45:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[40061]: Reconfiguring daemon mon.b on smithi045 2024-09-21T09:45:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[40061]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:25.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:25.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[29191]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:45:25.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:45:25.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:45:25.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:25.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[29191]: Reconfiguring daemon mon.b on smithi045 2024-09-21T09:45:25.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:24 smithi032 ceph-mon[29191]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:25.637 INFO:teuthology.orchestra.run.smithi045.stdout:Scheduled mgr update... 2024-09-21T09:45:26.189 DEBUG:teuthology.orchestra.run.smithi045:mgr.x> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x.service 2024-09-21T09:45:26.191 DEBUG:tasks.cephadm:set 0 configs 2024-09-21T09:45:26.192 INFO:tasks.cephadm:Deploying OSDs... 2024-09-21T09:45:26.192 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:45:26.192 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:45:26.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi032=y;smithi045=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: Saving service mgr spec with placement smithi032=y;smithi045=x;count:2 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: Deploying daemon mgr.x on smithi045 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[29191]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi032=y;smithi045=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: Saving service mgr spec with placement smithi032=y;smithi045=x;count:2 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: Deploying daemon mgr.x on smithi045 2024-09-21T09:45:26.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:25 smithi032 ceph-mon[40061]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:26.212 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:45:26.212 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_1 2024-09-21T09:45:26.212 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi032=y;smithi045=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: Saving service mgr spec with placement smithi032=y;smithi045=x;count:2 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:45:26.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:45:26.214 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:45:26.214 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:26.214 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: Deploying daemon mgr.x on smithi045 2024-09-21T09:45:26.214 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:25 smithi045 ceph-mon[33087]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:26.271 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:45:26.271 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:45:26.271 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 940 Links: 1 2024-09-21T09:45:26.271 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:45:26.271 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:45:26.271 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-09-21 09:45:12.571272109 +0000 2024-09-21T09:45:26.271 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-09-21 09:40:48.060840565 +0000 2024-09-21T09:45:26.271 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-09-21 09:40:48.060840565 +0000 2024-09-21T09:45:26.272 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-09-21 09:40:48.060840565 +0000 2024-09-21T09:45:26.272 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:45:26.341 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-09-21T09:45:26.341 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-09-21T09:45:26.341 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000116638 s, 4.4 MB/s 2024-09-21T09:45:26.343 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:45:26.403 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_2 2024-09-21T09:45:26.462 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:45:26.462 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:45:26.462 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 962 Links: 1 2024-09-21T09:45:26.462 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:45:26.463 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:45:26.463 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-09-21 09:45:12.572272088 +0000 2024-09-21T09:45:26.463 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-09-21 09:40:48.062840424 +0000 2024-09-21T09:45:26.463 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-09-21 09:40:48.062840424 +0000 2024-09-21T09:45:26.463 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-09-21 09:40:48.062840424 +0000 2024-09-21T09:45:26.463 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:45:26.530 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-09-21T09:45:26.530 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-09-21T09:45:26.530 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000125435 s, 4.1 MB/s 2024-09-21T09:45:26.532 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:45:26.592 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_3 2024-09-21T09:45:26.650 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:45:26.650 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:45:26.650 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 946 Links: 1 2024-09-21T09:45:26.650 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:45:26.651 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:45:26.651 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-09-21 09:45:12.572272088 +0000 2024-09-21T09:45:26.651 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-09-21 09:40:48.060840565 +0000 2024-09-21T09:45:26.651 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-09-21 09:40:48.060840565 +0000 2024-09-21T09:45:26.651 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-09-21 09:40:48.060840565 +0000 2024-09-21T09:45:26.651 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:45:26.717 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-09-21T09:45:26.717 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-09-21T09:45:26.717 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000123069 s, 4.2 MB/s 2024-09-21T09:45:26.719 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:45:26.778 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_4 2024-09-21T09:45:26.836 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:45:26.836 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:45:26.836 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 942 Links: 1 2024-09-21T09:45:26.836 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:45:26.836 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:45:26.837 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-09-21 09:45:12.573272067 +0000 2024-09-21T09:45:26.837 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-09-21 09:40:48.060840565 +0000 2024-09-21T09:45:26.837 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-09-21 09:40:48.060840565 +0000 2024-09-21T09:45:26.837 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-09-21 09:40:48.060840565 +0000 2024-09-21T09:45:26.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:45:26.903 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-09-21T09:45:26.903 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-09-21T09:45:26.904 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000121564 s, 4.2 MB/s 2024-09-21T09:45:26.905 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:45:26.964 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:45:26.964 DEBUG:teuthology.orchestra.run.smithi045:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:45:26.981 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:45:26.981 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_1 2024-09-21T09:45:27.038 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:45:27.038 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:45:27.038 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 5h/5d Inode: 959 Links: 1 2024-09-21T09:45:27.038 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:45:27.038 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:45:27.038 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2024-09-21 09:45:03.819995747 +0000 2024-09-21T09:45:27.038 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2024-09-21 09:40:47.438915251 +0000 2024-09-21T09:45:27.038 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2024-09-21 09:40:47.438915251 +0000 2024-09-21T09:45:27.038 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: 2024-09-21 09:40:47.438915251 +0000 2024-09-21T09:45:27.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:45:27.103 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2024-09-21T09:45:27.103 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2024-09-21T09:45:27.104 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 0.000107856 s, 4.7 MB/s 2024-09-21T09:45:27.105 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:45:27.162 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_2 2024-09-21T09:45:27.219 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:45:27.219 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:45:27.219 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 5h/5d Inode: 963 Links: 1 2024-09-21T09:45:27.219 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:45:27.219 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:45:27.219 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2024-09-21 09:45:03.819995747 +0000 2024-09-21T09:45:27.219 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2024-09-21 09:40:47.438915251 +0000 2024-09-21T09:45:27.219 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2024-09-21 09:40:47.438915251 +0000 2024-09-21T09:45:27.219 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: 2024-09-21 09:40:47.438915251 +0000 2024-09-21T09:45:27.219 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:45:27.283 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2024-09-21T09:45:27.283 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2024-09-21T09:45:27.283 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 9.5158e-05 s, 5.4 MB/s 2024-09-21T09:45:27.284 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:45:27.342 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_3 2024-09-21T09:45:27.398 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:45:27.398 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:45:27.399 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 5h/5d Inode: 964 Links: 1 2024-09-21T09:45:27.399 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:45:27.399 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:45:27.399 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2024-09-21 09:45:03.819995747 +0000 2024-09-21T09:45:27.399 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2024-09-21 09:40:47.438915251 +0000 2024-09-21T09:45:27.399 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2024-09-21 09:40:47.438915251 +0000 2024-09-21T09:45:27.399 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: 2024-09-21 09:40:47.438915251 +0000 2024-09-21T09:45:27.399 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:45:27.550 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2024-09-21T09:45:27.551 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2024-09-21T09:45:27.551 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 0.000105235 s, 4.9 MB/s 2024-09-21T09:45:27.552 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:45:27.568 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_4 2024-09-21T09:45:27.631 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:45:27.631 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:45:27.631 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-09-21T09:45:27.631 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:45:27.631 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:45:27.631 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2024-09-21 09:45:27.500625906 +0000 2024-09-21T09:45:27.631 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2024-09-21 09:40:47.440915219 +0000 2024-09-21T09:45:27.631 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2024-09-21 09:40:47.440915219 +0000 2024-09-21T09:45:27.631 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: 2024-09-21 09:40:47.440915219 +0000 2024-09-21T09:45:27.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:45:27.744 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2024-09-21T09:45:27.744 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2024-09-21T09:45:27.744 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 0.000108332 s, 4.7 MB/s 2024-09-21T09:45:27.745 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:45:27.761 INFO:tasks.cephadm:Deploying osd.0 on smithi032 with /dev/vg_nvme/lv_4... 2024-09-21T09:45:27.761 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:45:27.978 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:27 smithi045 ceph-mon[33087]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:27.979 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:45:28.051 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:27 smithi032 ceph-mon[29191]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:28.051 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:27 smithi032 ceph-mon[40061]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:28.232 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 podman[34925]: 2024-09-21 09:45:28.20482165 +0000 UTC m=+0.341764046 container start ec216715bfa0c7a82d47671773ff553ba994973f81efd86470d575247c108ea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, RELEASE=reef-6a1bdf8, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T09:45:28.232 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 ceph-mgr[34944]: -- 172.21.15.45:0/1273128114 <== mon.1 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55de5f4b0d00 con 0x55de5f485000 2024-09-21T09:45:28.534 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 bash[34925]: ec216715bfa0c7a82d47671773ff553ba994973f81efd86470d575247c108ea1 2024-09-21T09:45:28.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 systemd[1]: Started Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:45:28.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:28.328+0000 7fc81aca6040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:45:28.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:28.387+0000 7fc81aca6040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:45:28.941 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:28.554+0000 7fc81aca6040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:45:28.941 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:28.626+0000 7fc81aca6040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:45:28.941 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:28.794+0000 7fc81aca6040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:45:28.941 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:28.938+0000 7fc81aca6040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:45:29.529 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:29.266+0000 7fc81aca6040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:45:29.529 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:29.329+0000 7fc81aca6040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:45:29.529 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:29 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.529 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:29 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.529 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:29 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.529 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:29 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.529 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:29 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:29.600 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.600 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.600 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.600 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.600 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:29.600 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.601 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.601 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.601 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:29.601 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:29 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:29.784 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:29.526+0000 7fc81aca6040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:45:30.466 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:45:30.490 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi032:vg_nvme/lv_4 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: Reconfiguring daemon mgr.y on smithi032 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:30.736 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: Reconfiguring daemon mgr.y on smithi032 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:30.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.784 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:30.434+0000 7fc81aca6040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:45:30.784 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:30.503+0000 7fc81aca6040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: Reconfiguring daemon mgr.y on smithi032 2024-09-21T09:45:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:30.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:31.481 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:45:31.517 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:31.249+0000 7fc81aca6040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:45:31.517 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:31.368+0000 7fc81aca6040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:45:31.784 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:31.514+0000 7fc81aca6040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:45:31.785 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:31.578+0000 7fc81aca6040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:45:31.785 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:31.734+0000 7fc81aca6040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:45:32.097 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:31.804+0000 7fc81aca6040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:45:32.097 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:31.866+0000 7fc81aca6040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:45:32.097 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:32.027+0000 7fc81aca6040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:45:32.097 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:32.094+0000 7fc81aca6040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:45:32.373 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:32.163+0000 7fc81aca6040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:45:32.373 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:32.249+0000 7fc81aca6040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:45:32.373 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:32.309+0000 7fc81aca6040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:45:32.373 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:32.370+0000 7fc81aca6040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:45:32.549 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:32.549 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:32.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:32.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:32.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:32.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:32.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:32.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:32.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:32 smithi045 ceph-mon[33087]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:32.802 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:32.802 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:32.802 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:32.802 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:32.802 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[29191]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:32.802 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:32.802 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:32.803 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:32.803 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:32.803 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:32.803 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:32.803 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:32 smithi032 ceph-mon[40061]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:33.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:45:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:45:33.179+0000 7fc81aca6040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:45:33.798 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:33.798 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:33.798 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:33.798 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[40061]: Standby manager daemon x started 2024-09-21T09:45:33.798 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:45:33.798 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:45:33.798 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:45:33.798 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:45:33.798 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:33.799 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:33.799 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:33.799 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[29191]: Standby manager daemon x started 2024-09-21T09:45:33.799 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:45:33.799 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:45:33.799 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:45:33.799 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:33 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:45:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:33 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:33 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:33 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:33 smithi045 ceph-mon[33087]: Standby manager daemon x started 2024-09-21T09:45:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:33 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:45:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:33 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:45:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:33 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:45:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:33 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/517313907' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:45:34.913 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:34 smithi032 ceph-mon[29191]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:34.913 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:34 smithi032 ceph-mon[29191]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-21T09:45:34.914 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:34 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:45:34.914 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:34 smithi032 ceph-mon[29191]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:34.914 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:34 smithi032 ceph-mon[40061]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:34.914 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:34 smithi032 ceph-mon[40061]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-21T09:45:34.914 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:34 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:45:34.914 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:34 smithi032 ceph-mon[40061]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:35.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:34 smithi045 ceph-mon[33087]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:35.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:34 smithi045 ceph-mon[33087]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-21T09:45:35.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:34 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:45:35.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:34 smithi045 ceph-mon[33087]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1709703228' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f6f5dda-c134-4283-a91d-cf501ef19062"}]: dispatch 2024-09-21T09:45:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f6f5dda-c134-4283-a91d-cf501ef19062"}]: dispatch 2024-09-21T09:45:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0f6f5dda-c134-4283-a91d-cf501ef19062"}]': finished 2024-09-21T09:45:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[40061]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:45:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1709703228' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f6f5dda-c134-4283-a91d-cf501ef19062"}]: dispatch 2024-09-21T09:45:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f6f5dda-c134-4283-a91d-cf501ef19062"}]: dispatch 2024-09-21T09:45:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0f6f5dda-c134-4283-a91d-cf501ef19062"}]': finished 2024-09-21T09:45:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[29191]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:45:35.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:35 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:35 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1709703228' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f6f5dda-c134-4283-a91d-cf501ef19062"}]: dispatch 2024-09-21T09:45:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:35 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f6f5dda-c134-4283-a91d-cf501ef19062"}]: dispatch 2024-09-21T09:45:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:35 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0f6f5dda-c134-4283-a91d-cf501ef19062"}]': finished 2024-09-21T09:45:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:35 smithi045 ceph-mon[33087]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:45:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:35 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:36.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:36 smithi032 ceph-mon[29191]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:36.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:36 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3779860954' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:36.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:36 smithi032 ceph-mon[40061]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:36.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:36 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3779860954' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:37.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:36 smithi045 ceph-mon[33087]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:37.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:36 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3779860954' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:38.189 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:37 smithi032 ceph-mon[40061]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:38.190 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:37 smithi032 ceph-mon[29191]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:38.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:37 smithi045 ceph-mon[33087]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:39.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:39 smithi032 ceph-mon[29191]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:39 smithi032 ceph-mon[40061]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:40.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:39 smithi045 ceph-mon[33087]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:41.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:41 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:45:41.968 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:41 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:41.969 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:41 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:45:41.969 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:41 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:42.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:41 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:45:42.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:41 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:42.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:42 smithi032 ceph-mon[29191]: Deploying daemon osd.0 on smithi032 2024-09-21T09:45:42.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:42 smithi032 ceph-mon[29191]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:42 smithi032 ceph-mon[40061]: Deploying daemon osd.0 on smithi032 2024-09-21T09:45:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:42 smithi032 ceph-mon[40061]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:43.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:42 smithi045 ceph-mon[33087]: Deploying daemon osd.0 on smithi032 2024-09-21T09:45:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:42 smithi045 ceph-mon[33087]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:43.980 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:43 smithi032 ceph-mon[29191]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:43.980 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:43 smithi032 ceph-mon[40061]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:43 smithi045 ceph-mon[33087]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:46.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:45 smithi032 ceph-mon[29191]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:46.173 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:45 smithi032 ceph-mon[40061]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:46.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:45 smithi045 ceph-mon[33087]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:46 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:46 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:46 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:47.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:46 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:47.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:46 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:47.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:46 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:47.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:47 smithi032 ceph-mon[29191]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:47.935 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:47 smithi032 ceph-mon[40061]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:48.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:47 smithi045 ceph-mon[33087]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:49.639 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 0 on host 'smithi032' 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='osd.0 [v2:172.21.15.32:6802/2765215923,v1:172.21.15.32:6803/2765215923]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:45:49.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[29191]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='osd.0 [v2:172.21.15.32:6802/2765215923,v1:172.21.15.32:6803/2765215923]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:45:49.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:49 smithi032 ceph-mon[40061]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:45:50.192 DEBUG:teuthology.orchestra.run.smithi032:osd.0> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T09:45:50.194 INFO:tasks.cephadm:Deploying osd.1 on smithi032 with /dev/vg_nvme/lv_3... 2024-09-21T09:45:50.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:45:50.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='osd.0 [v2:172.21.15.32:6802/2765215923,v1:172.21.15.32:6803/2765215923]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:45:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:49 smithi045 ceph-mon[33087]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[29191]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[29191]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[29191]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[29191]: from='osd.0 [v2:172.21.15.32:6802/2765215923,v1:172.21.15.32:6803/2765215923]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[29191]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[40061]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[40061]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[40061]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:45:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[40061]: from='osd.0 [v2:172.21.15.32:6802/2765215923,v1:172.21.15.32:6803/2765215923]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:45:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:50 smithi032 ceph-mon[40061]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:45:51.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:50 smithi045 ceph-mon[33087]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:50 smithi045 ceph-mon[33087]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:45:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:50 smithi045 ceph-mon[33087]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:45:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:50 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:50 smithi045 ceph-mon[33087]: from='osd.0 [v2:172.21.15.32:6802/2765215923,v1:172.21.15.32:6803/2765215923]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:45:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:50 smithi045 ceph-mon[33087]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:45:51.802 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:45:52.168 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:45:51 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[43968]: 2024-09-21T09:45:51.904+0000 7f479de2b640 -1 osd.0 0 waiting for initial osdmap 2024-09-21T09:45:52.168 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:45:51 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[43968]: 2024-09-21T09:45:51.906+0000 7f4795c4c640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:45:52.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[29191]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[29191]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[29191]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[29191]: from='osd.0 ' entity='osd.0' 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[40061]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[40061]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[40061]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:52.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:52 smithi032 ceph-mon[40061]: from='osd.0 ' entity='osd.0' 2024-09-21T09:45:53.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:52 smithi045 ceph-mon[33087]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:45:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:52 smithi045 ceph-mon[33087]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:45:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:52 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:52 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:52 smithi045 ceph-mon[33087]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:45:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:52 smithi045 ceph-mon[33087]: from='osd.0 ' entity='osd.0' 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: purged_snaps scrub starts 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: purged_snaps scrub ok 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: osd.0 [v2:172.21.15.32:6802/2765215923,v1:172.21.15.32:6803/2765215923] boot 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: purged_snaps scrub starts 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: purged_snaps scrub ok 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: osd.0 [v2:172.21.15.32:6802/2765215923,v1:172.21.15.32:6803/2765215923] boot 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:53.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:53.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:53.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:53 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: purged_snaps scrub starts 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: purged_snaps scrub ok 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: osd.0 [v2:172.21.15.32:6802/2765215923,v1:172.21.15.32:6803/2765215923] boot 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:53 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:45:54.737 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:45:54.756 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi032:vg_nvme/lv_3 2024-09-21T09:45:54.955 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:45:54.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:54 smithi032 ceph-mon[29191]: Detected new or changed devices on smithi032 2024-09-21T09:45:54.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:54 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi032 to 16068M 2024-09-21T09:45:54.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:54 smithi032 ceph-mon[29191]: pgmap v40: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:45:54.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:54 smithi032 ceph-mon[40061]: Detected new or changed devices on smithi032 2024-09-21T09:45:54.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:54 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi032 to 16068M 2024-09-21T09:45:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:54 smithi032 ceph-mon[40061]: pgmap v40: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:45:55.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:54 smithi045 ceph-mon[33087]: Detected new or changed devices on smithi032 2024-09-21T09:45:55.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:54 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi032 to 16068M 2024-09-21T09:45:55.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:54 smithi045 ceph-mon[33087]: pgmap v40: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:45:55.683 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:55 smithi032 ceph-mon[29191]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:45:55.683 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:55 smithi032 ceph-mon[40061]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:45:56.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:55 smithi045 ceph-mon[33087]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:45:56.943 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:56 smithi032 ceph-mon[40061]: pgmap v42: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:45:56.944 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:56 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:56.944 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:56 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:56.944 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:56 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:56.944 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:56 smithi032 ceph-mon[29191]: pgmap v42: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:45:56.944 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:56.944 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:56.944 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:56 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:57.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:56 smithi045 ceph-mon[33087]: pgmap v42: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:45:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:56 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:56 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:56 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:57.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:57 smithi032 ceph-mon[29191]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:57.933 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:57 smithi032 ceph-mon[40061]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:58.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:57 smithi045 ceph-mon[33087]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:58.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[29191]: pgmap v43: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1142631726' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdaa43fd-b969-4d06-b6eb-97b415c4e118"}]: dispatch 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdaa43fd-b969-4d06-b6eb-97b415c4e118"}]: dispatch 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bdaa43fd-b969-4d06-b6eb-97b415c4e118"}]': finished 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[29191]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[40061]: pgmap v43: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1142631726' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdaa43fd-b969-4d06-b6eb-97b415c4e118"}]: dispatch 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdaa43fd-b969-4d06-b6eb-97b415c4e118"}]: dispatch 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bdaa43fd-b969-4d06-b6eb-97b415c4e118"}]': finished 2024-09-21T09:45:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[40061]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:45:58.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:58 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:45:59.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:58 smithi045 ceph-mon[33087]: pgmap v43: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:45:59.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:58 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1142631726' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdaa43fd-b969-4d06-b6eb-97b415c4e118"}]: dispatch 2024-09-21T09:45:59.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:58 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdaa43fd-b969-4d06-b6eb-97b415c4e118"}]: dispatch 2024-09-21T09:45:59.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:58 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bdaa43fd-b969-4d06-b6eb-97b415c4e118"}]': finished 2024-09-21T09:45:59.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:58 smithi045 ceph-mon[33087]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:45:59.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:58 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:45:59.952 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:45:59 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/2247087286' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:59.953 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:45:59 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2247087286' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:00.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:45:59 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/2247087286' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:00.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:00 smithi032 ceph-mon[29191]: pgmap v45: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:00.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:00 smithi032 ceph-mon[40061]: pgmap v45: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:01.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:00 smithi045 ceph-mon[33087]: pgmap v45: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:02.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:01 smithi032 ceph-mon[29191]: pgmap v46: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:02.076 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:01 smithi032 ceph-mon[40061]: pgmap v46: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:02.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:01 smithi045 ceph-mon[33087]: pgmap v46: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:03 smithi032 ceph-mon[29191]: pgmap v47: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:03 smithi032 ceph-mon[40061]: pgmap v47: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:04.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:03 smithi045 ceph-mon[33087]: pgmap v47: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:05.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:04 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:46:05.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:04 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:05.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:04 smithi032 ceph-mon[29191]: Deploying daemon osd.1 on smithi032 2024-09-21T09:46:05.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:04 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:46:05.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:04 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:05.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:04 smithi032 ceph-mon[40061]: Deploying daemon osd.1 on smithi032 2024-09-21T09:46:05.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:04 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:46:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:04 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:04 smithi045 ceph-mon[33087]: Deploying daemon osd.1 on smithi032 2024-09-21T09:46:06.169 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:05 smithi032 ceph-mon[29191]: pgmap v48: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:06.170 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:05 smithi032 ceph-mon[40061]: pgmap v48: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:06.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:05 smithi045 ceph-mon[33087]: pgmap v48: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:08.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:07 smithi032 ceph-mon[29191]: pgmap v49: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:08.191 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:07 smithi032 ceph-mon[40061]: pgmap v49: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:08.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:07 smithi045 ceph-mon[33087]: pgmap v49: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:09.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:09.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:09.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:09 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:09 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:09 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:11.004 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:10 smithi032 ceph-mon[29191]: pgmap v50: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:11.005 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:10 smithi032 ceph-mon[40061]: pgmap v50: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:10 smithi045 ceph-mon[33087]: pgmap v50: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:12.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:11 smithi032 ceph-mon[29191]: pgmap v51: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:11 smithi032 ceph-mon[40061]: pgmap v51: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:12.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:11 smithi045 ceph-mon[33087]: pgmap v51: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:13.033 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 1 on host 'smithi032' 2024-09-21T09:46:13.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:13.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:13.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[29191]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:46:13.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:13.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[40061]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:13 smithi045 ceph-mon[33087]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:46:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:13.666 DEBUG:teuthology.orchestra.run.smithi032:osd.1> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.1.service 2024-09-21T09:46:13.669 INFO:tasks.cephadm:Deploying osd.2 on smithi032 with /dev/vg_nvme/lv_2... 2024-09-21T09:46:13.669 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:46:14.356 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[29191]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:46:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[29191]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:46:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[29191]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:46:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[29191]: pgmap v53: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[40061]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:46:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[40061]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:46:14.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:14.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[40061]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:46:14.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-mon[40061]: pgmap v53: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:14.458 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[49824]: 2024-09-21T09:46:14.235+0000 7f40a6dc9640 -1 osd.1 0 waiting for initial osdmap 2024-09-21T09:46:14.458 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:14 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[49824]: 2024-09-21T09:46:14.238+0000 7f40a2bf2640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:46:14.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:14 smithi045 ceph-mon[33087]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:46:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:14 smithi045 ceph-mon[33087]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:46:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:14 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:14 smithi045 ceph-mon[33087]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:46:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:14 smithi045 ceph-mon[33087]: pgmap v53: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:15.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:15 smithi045 ceph-mon[33087]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:46:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:15 smithi045 ceph-mon[33087]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:46:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:15 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:15 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:15.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:15 smithi032 ceph-mon[29191]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:46:15.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:15 smithi032 ceph-mon[29191]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:46:15.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:15 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:15.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:15 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:15.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:15 smithi032 ceph-mon[40061]: from='osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:46:15.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:15 smithi032 ceph-mon[40061]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:46:15.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:15 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:15.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:15 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:16 smithi045 ceph-mon[33087]: purged_snaps scrub starts 2024-09-21T09:46:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:16 smithi045 ceph-mon[33087]: purged_snaps scrub ok 2024-09-21T09:46:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:16 smithi045 ceph-mon[33087]: osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383] boot 2024-09-21T09:46:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:16 smithi045 ceph-mon[33087]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:46:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:16 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:16 smithi045 ceph-mon[33087]: pgmap v56: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:16.602 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[29191]: purged_snaps scrub starts 2024-09-21T09:46:16.602 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[29191]: purged_snaps scrub ok 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[29191]: osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383] boot 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[29191]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[29191]: pgmap v56: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[40061]: purged_snaps scrub starts 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[40061]: purged_snaps scrub ok 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[40061]: osd.1 [v2:172.21.15.32:6810/3808018383,v1:172.21.15.32:6811/3808018383] boot 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[40061]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:46:16.603 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:16 smithi032 ceph-mon[40061]: pgmap v56: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:46:17.511 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:17 smithi032 ceph-mon[29191]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:46:17.511 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:17 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:17.511 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:17 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:17.511 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:17 smithi032 ceph-mon[40061]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:46:17.512 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:17 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:17.512 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:17 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:17.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:17 smithi045 ceph-mon[33087]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:46:17.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:17 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:17 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:18.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[40061]: Detected new or changed devices on smithi032 2024-09-21T09:46:18.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:18.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:18.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi032 to 8034M 2024-09-21T09:46:18.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:18.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[40061]: pgmap v58: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[29191]: Detected new or changed devices on smithi032 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi032 to 8034M 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:18.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:18 smithi032 ceph-mon[29191]: pgmap v58: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:18.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:18 smithi045 ceph-mon[33087]: Detected new or changed devices on smithi032 2024-09-21T09:46:18.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:18.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:18.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:18 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi032 to 8034M 2024-09-21T09:46:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:18 smithi045 ceph-mon[33087]: pgmap v58: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:18.651 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:46:18.668 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi032:vg_nvme/lv_2 2024-09-21T09:46:18.869 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:20.067 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:19 smithi032 ceph-mon[29191]: pgmap v59: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:20.068 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:19 smithi032 ceph-mon[40061]: pgmap v59: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:20.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:19 smithi045 ceph-mon[33087]: pgmap v59: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:21.101 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:20 smithi032 ceph-mon[29191]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:21.101 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:21.102 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:21.102 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:21.102 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:20 smithi032 ceph-mon[40061]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:21.102 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:21.102 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:21.102 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:21.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:20 smithi045 ceph-mon[33087]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:21.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:21 smithi032 ceph-mon[29191]: pgmap v60: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:21.965 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:21 smithi032 ceph-mon[40061]: pgmap v60: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:22.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:21 smithi045 ceph-mon[33087]: pgmap v60: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:22 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3379258840' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1f12d9f2-90b0-44f6-8432-0b398194b3b3"}]: dispatch 2024-09-21T09:46:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:22 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3379258840' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1f12d9f2-90b0-44f6-8432-0b398194b3b3"}]': finished 2024-09-21T09:46:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:22 smithi032 ceph-mon[29191]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:46:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:22 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:23.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:22 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3379258840' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1f12d9f2-90b0-44f6-8432-0b398194b3b3"}]: dispatch 2024-09-21T09:46:23.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:22 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3379258840' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1f12d9f2-90b0-44f6-8432-0b398194b3b3"}]': finished 2024-09-21T09:46:23.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:22 smithi032 ceph-mon[40061]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:46:23.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:22 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:23.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:22 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3379258840' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1f12d9f2-90b0-44f6-8432-0b398194b3b3"}]: dispatch 2024-09-21T09:46:23.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:22 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3379258840' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1f12d9f2-90b0-44f6-8432-0b398194b3b3"}]': finished 2024-09-21T09:46:23.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:22 smithi045 ceph-mon[33087]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:46:23.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:22 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:23 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1352982850' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:23 smithi032 ceph-mon[29191]: pgmap v62: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:23 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1352982850' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:23 smithi032 ceph-mon[40061]: pgmap v62: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:23 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1352982850' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:23 smithi045 ceph-mon[33087]: pgmap v62: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:26.104 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:25 smithi032 ceph-mon[40061]: pgmap v63: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:26.105 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:25 smithi032 ceph-mon[29191]: pgmap v63: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:26.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:25 smithi045 ceph-mon[33087]: pgmap v63: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:28.010 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:27 smithi032 ceph-mon[29191]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:28.011 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:27 smithi032 ceph-mon[40061]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:28.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:27 smithi045 ceph-mon[33087]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:29.129 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:46:29.129 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:29.129 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:28 smithi032 ceph-mon[29191]: Deploying daemon osd.2 on smithi032 2024-09-21T09:46:29.129 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:46:29.129 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:29.129 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:28 smithi032 ceph-mon[40061]: Deploying daemon osd.2 on smithi032 2024-09-21T09:46:29.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:46:29.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:29.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:28 smithi045 ceph-mon[33087]: Deploying daemon osd.2 on smithi032 2024-09-21T09:46:30.185 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:29 smithi032 ceph-mon[40061]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:30.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:29 smithi032 ceph-mon[29191]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:30.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:29 smithi045 ceph-mon[33087]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:31.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:31 smithi032 ceph-mon[29191]: pgmap v66: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:31 smithi032 ceph-mon[40061]: pgmap v66: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:32.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:31 smithi045 ceph-mon[33087]: pgmap v66: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:33.671 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:33 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:33.671 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:33 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:33.671 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:33 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:33.671 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:33.672 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:33.672 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:33.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:33 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:33.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:33 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:33.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:33 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:34.686 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:34 smithi032 ceph-mon[29191]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:34.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:34 smithi032 ceph-mon[40061]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:34.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:34 smithi045 ceph-mon[33087]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:36.141 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:35 smithi032 ceph-mon[40061]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:36.141 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:35 smithi032 ceph-mon[29191]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:36.173 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 2 on host 'smithi032' 2024-09-21T09:46:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:35 smithi045 ceph-mon[33087]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:36.825 DEBUG:teuthology.orchestra.run.smithi032:osd.2> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.2.service 2024-09-21T09:46:36.827 INFO:tasks.cephadm:Deploying osd.3 on smithi032 with /dev/vg_nvme/lv_1... 2024-09-21T09:46:36.828 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:46:37.084 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[29191]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[40061]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:37 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:37 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:37 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:37 smithi045 ceph-mon[33087]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:46:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:37 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:37 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:38.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[29191]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:46:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[29191]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:46:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[29191]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:46:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[29191]: pgmap v70: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[40061]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:46:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[40061]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:46:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[40061]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:46:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-mon[40061]: pgmap v70: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:38.532 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[55639]: 2024-09-21T09:46:38.450+0000 7f8dab4f2640 -1 osd.2 0 waiting for initial osdmap 2024-09-21T09:46:38.532 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[55639]: 2024-09-21T09:46:38.453+0000 7f8da2afc640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:46:38.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:38 smithi045 ceph-mon[33087]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:46:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:38 smithi045 ceph-mon[33087]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:46:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:38 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:38 smithi045 ceph-mon[33087]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:46:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:38 smithi045 ceph-mon[33087]: pgmap v70: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:39.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[29191]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:46:39.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[29191]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:46:39.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:39.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:39.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[29191]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' 2024-09-21T09:46:39.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[40061]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:46:39.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[40061]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:46:39.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:39.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:39.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:39 smithi032 ceph-mon[40061]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' 2024-09-21T09:46:39.491 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:46:39.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi032:vg_nvme/lv_1 2024-09-21T09:46:39.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:39 smithi045 ceph-mon[33087]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:46:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:39 smithi045 ceph-mon[33087]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:46:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:39 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:39 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:39 smithi045 ceph-mon[33087]: from='osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298]' entity='osd.2' 2024-09-21T09:46:40.435 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[29191]: purged_snaps scrub starts 2024-09-21T09:46:40.435 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[29191]: purged_snaps scrub ok 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[29191]: osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298] boot 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[29191]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[29191]: pgmap v73: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[40061]: purged_snaps scrub starts 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[40061]: purged_snaps scrub ok 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[40061]: osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298] boot 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[40061]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:40.436 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:40 smithi032 ceph-mon[40061]: pgmap v73: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:40.485 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:40 smithi045 ceph-mon[33087]: purged_snaps scrub starts 2024-09-21T09:46:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:40 smithi045 ceph-mon[33087]: purged_snaps scrub ok 2024-09-21T09:46:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:40 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:40 smithi045 ceph-mon[33087]: osd.2 [v2:172.21.15.32:6818/2468805298,v1:172.21.15.32:6819/2468805298] boot 2024-09-21T09:46:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:40 smithi045 ceph-mon[33087]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:46:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:40 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:46:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:40 smithi045 ceph-mon[33087]: pgmap v73: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:46:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:41 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:46:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:41 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:41 smithi032 ceph-mon[40061]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:46:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:41 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:46:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:41 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:46:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:41 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:41 smithi032 ceph-mon[29191]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:46:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:41 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:46:41.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:41 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:46:41.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:41 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:41 smithi045 ceph-mon[33087]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:46:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:41 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:46:42.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:42.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:46:42.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:42.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:46:42.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:42.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:42 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:42.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:42.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:42.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:42.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:42.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:42.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:42 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:43.206 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:46:42 smithi032 sudo[59741]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:46:43.206 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:46:42 smithi032 sudo[59741]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:43.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:46:42 smithi032 sudo[59741]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:43.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:46:42 smithi032 sudo[59741]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:43.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[40061]: Detected new or changed devices on smithi032 2024-09-21T09:46:43.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi032 to 5356M 2024-09-21T09:46:43.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[40061]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:43.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:43.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:43.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:43.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[40061]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:46:43.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:43.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[29191]: Detected new or changed devices on smithi032 2024-09-21T09:46:43.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi032 to 5356M 2024-09-21T09:46:43.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[29191]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:43.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:43.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:43.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:43.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[29191]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:46:43.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:43 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:43.708 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:46:43 smithi032 sudo[59818]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:46:43.708 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:46:43 smithi032 sudo[59818]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:43.708 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:46:43 smithi032 sudo[59818]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:43.708 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 09:46:43 smithi032 sudo[59818]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:43.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:43 smithi045 ceph-mon[33087]: Detected new or changed devices on smithi032 2024-09-21T09:46:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:43 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi032 to 5356M 2024-09-21T09:46:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:43 smithi045 ceph-mon[33087]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:43 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:43 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:43 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:43 smithi045 ceph-mon[33087]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:46:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:43 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:44.081 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:43 smithi032 sudo[59824]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:46:44.081 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:43 smithi032 sudo[59824]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:44.081 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:43 smithi032 sudo[59824]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:44.081 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:43 smithi032 sudo[59824]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:44.368 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:44 smithi032 sudo[59848]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:46:44.369 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:44 smithi032 sudo[59848]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:44.369 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:44 smithi032 sudo[59848]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:44.369 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 09:46:44 smithi032 sudo[59848]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:44.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:44 smithi032 ceph-mon[29191]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:44.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:44 smithi032 ceph-mon[40061]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:44.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:44 smithi045 ceph-mon[33087]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:45.206 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:44 smithi032 sudo[59874]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:46:45.206 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:44 smithi032 sudo[59874]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:45.206 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:44 smithi032 sudo[59874]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:45.207 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:44 smithi032 sudo[59874]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:45.645 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:45 smithi032 sudo[59906]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:46:45.645 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:45 smithi032 sudo[59906]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:45.645 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:45 smithi032 sudo[59906]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:45.645 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 09:46:45 smithi032 sudo[59906]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:45.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:45 smithi032 sudo[59939]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:46:45.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:45 smithi032 sudo[59939]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:45 smithi032 sudo[59939]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:45 smithi032 sudo[59939]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:46.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 sudo[59988]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 sudo[59988]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 sudo[59988]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 sudo[59988]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[29191]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1617996538' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a568fd5-5fd9-46f8-b3af-9fa196ed617b"}]: dispatch 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1617996538' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a568fd5-5fd9-46f8-b3af-9fa196ed617b"}]': finished 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[29191]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[29191]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[40061]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:46:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1617996538' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a568fd5-5fd9-46f8-b3af-9fa196ed617b"}]: dispatch 2024-09-21T09:46:46.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1617996538' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a568fd5-5fd9-46f8-b3af-9fa196ed617b"}]': finished 2024-09-21T09:46:46.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[40061]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:46:46.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:46:46.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:46 smithi032 ceph-mon[40061]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:46:46.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 ceph-mon[33087]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:46:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1617996538' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a568fd5-5fd9-46f8-b3af-9fa196ed617b"}]: dispatch 2024-09-21T09:46:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1617996538' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a568fd5-5fd9-46f8-b3af-9fa196ed617b"}]': finished 2024-09-21T09:46:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 ceph-mon[33087]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:46:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:46:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 ceph-mon[33087]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:46:47.029 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 sudo[35596]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:46:47.029 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 sudo[35596]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:47.029 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 sudo[35596]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:47.029 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:46 smithi045 sudo[35596]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:47.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 sudo[35599]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:46:47.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 sudo[35599]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 sudo[35599]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 sudo[35599]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:47.786 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 sudo[60732]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:46:47.786 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 sudo[60732]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:47.786 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 sudo[60732]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:47.786 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 sudo[60732]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/550471554' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 89 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:46:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 sudo[60838]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 sudo[60838]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 sudo[60838]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 sudo[60838]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/550471554' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 89 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:46:48.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/550471554' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 89 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:46:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:48.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:46:48.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:46:48.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:46:49.068 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:48 smithi032 ceph-mon[40061]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:46:49.068 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:48 smithi032 ceph-mon[40061]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:49.069 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:48 smithi032 ceph-mon[40061]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:46:49.069 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:48 smithi032 ceph-mon[29191]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:46:49.069 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:48 smithi032 ceph-mon[29191]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:49.069 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:48 smithi032 ceph-mon[29191]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:46:49.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:48 smithi045 ceph-mon[33087]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:46:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:48 smithi045 ceph-mon[33087]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:48 smithi045 ceph-mon[33087]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:46:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:49 smithi045 ceph-mon[33087]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 91 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:50.310 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:49 smithi032 ceph-mon[29191]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 91 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:50.310 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:49 smithi032 ceph-mon[40061]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 91 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:46:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:50 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:46:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:50 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:50 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:46:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:50 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:51.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:50 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:46:51.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:50 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:52.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:51 smithi045 ceph-mon[33087]: Deploying daemon osd.3 on smithi032 2024-09-21T09:46:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:51 smithi045 ceph-mon[33087]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-21T09:46:52.308 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:51 smithi032 ceph-mon[29191]: Deploying daemon osd.3 on smithi032 2024-09-21T09:46:52.308 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:51 smithi032 ceph-mon[29191]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-21T09:46:52.308 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:51 smithi032 ceph-mon[40061]: Deploying daemon osd.3 on smithi032 2024-09-21T09:46:52.308 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:51 smithi032 ceph-mon[40061]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-21T09:46:53.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:53 smithi032 ceph-mon[29191]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-21T09:46:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:53 smithi032 ceph-mon[40061]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-21T09:46:54.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:53 smithi045 ceph-mon[33087]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-21T09:46:56.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:55 smithi032 ceph-mon[29191]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T09:46:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:55 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:55 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:55 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:55 smithi032 ceph-mon[40061]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T09:46:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:55 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:55 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:55 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:55 smithi045 ceph-mon[33087]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T09:46:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:55 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:55 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:55 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:57 smithi032 ceph-mon[29191]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T09:46:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:57 smithi032 ceph-mon[40061]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T09:46:58.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:57 smithi045 ceph-mon[33087]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T09:46:59.126 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 3 on host 'smithi032' 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='osd.3 [v2:172.21.15.32:6826/2765373470,v1:172.21.15.32:6827/2765373470]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:59.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:59.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='osd.3 [v2:172.21.15.32:6826/2765373470,v1:172.21.15.32:6827/2765373470]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:46:59.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:46:59.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:59.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:46:59 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.762 DEBUG:teuthology.orchestra.run.smithi032:osd.3> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T09:46:59.764 INFO:tasks.cephadm:Deploying osd.4 on smithi045 with /dev/vg_nvme/lv_4... 2024-09-21T09:46:59.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:46:59.796 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.796 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.796 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:59.796 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:59.797 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.797 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='osd.3 [v2:172.21.15.32:6826/2765373470,v1:172.21.15.32:6827/2765373470]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:46:59.797 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:46:59.797 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:59.797 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.797 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:46:59 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:46:59.977 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:47:01.145 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[40061]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:47:01.145 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[40061]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:47:01.145 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[40061]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:47:01.145 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:01.145 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[40061]: from='osd.3 [v2:172.21.15.32:6826/2765373470,v1:172.21.15.32:6827/2765373470]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:47:01.146 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[40061]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:47:01.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[29191]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:47:01.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[29191]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:47:01.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[29191]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:47:01.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:01.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[29191]: from='osd.3 [v2:172.21.15.32:6826/2765373470,v1:172.21.15.32:6827/2765373470]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:47:01.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-mon[29191]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:47:01.146 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[61571]: 2024-09-21T09:47:00.874+0000 7f17c19fa640 -1 osd.3 0 waiting for initial osdmap 2024-09-21T09:47:01.147 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 09:47:00 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[61571]: 2024-09-21T09:47:00.880+0000 7f17bd823640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:47:01.232 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:00 smithi045 ceph-mon[33087]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:47:01.232 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:00 smithi045 ceph-mon[33087]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:47:01.232 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:00 smithi045 ceph-mon[33087]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:47:01.232 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:00 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:01.232 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:00 smithi045 ceph-mon[33087]: from='osd.3 [v2:172.21.15.32:6826/2765373470,v1:172.21.15.32:6827/2765373470]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:47:01.232 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:00 smithi045 ceph-mon[33087]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-09-21T09:47:02.121 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:01 smithi045 ceph-mon[33087]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:47:02.121 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:01 smithi045 ceph-mon[33087]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:47:02.121 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:01 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:02.121 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:01 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:02.121 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:01 smithi045 ceph-mon[33087]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:47:02.170 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[29191]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:47:02.170 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[29191]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:47:02.170 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:02.170 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:02.170 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[29191]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:47:02.170 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[40061]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-09-21T09:47:02.170 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[40061]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:47:02.170 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:02.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:02.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:01 smithi032 ceph-mon[40061]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:47:02.382 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:47:02.402 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi045:vg_nvme/lv_4 2024-09-21T09:47:02.647 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:47:02.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: purged_snaps scrub starts 2024-09-21T09:47:02.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: purged_snaps scrub ok 2024-09-21T09:47:02.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: osd.3 [v2:172.21.15.32:6826/2765373470,v1:172.21.15.32:6827/2765373470] boot 2024-09-21T09:47:02.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:47:02.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: Detected new or changed devices on smithi032 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi032 to 4017M 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:03.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: purged_snaps scrub starts 2024-09-21T09:47:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: purged_snaps scrub ok 2024-09-21T09:47:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: osd.3 [v2:172.21.15.32:6826/2765373470,v1:172.21.15.32:6827/2765373470] boot 2024-09-21T09:47:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:47:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: Detected new or changed devices on smithi032 2024-09-21T09:47:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi032 to 4017M 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: purged_snaps scrub starts 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: purged_snaps scrub ok 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: osd.3 [v2:172.21.15.32:6826/2765373470,v1:172.21.15.32:6827/2765373470] boot 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: Detected new or changed devices on smithi032 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi032 to 4017M 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:03.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:03.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:03.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:04.112 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:03 smithi045 ceph-mon[33087]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:47:04.113 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:03 smithi045 ceph-mon[33087]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:04.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:03 smithi032 ceph-mon[29191]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:47:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:03 smithi032 ceph-mon[29191]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:03 smithi032 ceph-mon[40061]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:47:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:03 smithi032 ceph-mon[40061]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:05.191 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:04 smithi045 ceph-mon[33087]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:05.191 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:04 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:47:05.191 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:04 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:47:05.191 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:04 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:05.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:04 smithi032 ceph-mon[29191]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:05.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:04 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:47:05.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:04 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:47:05.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:04 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:04 smithi032 ceph-mon[40061]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:04 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:47:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:04 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:47:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:04 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:06.172 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:05 smithi045 ceph-mon[33087]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:06.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:05 smithi032 ceph-mon[29191]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:05 smithi032 ceph-mon[40061]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:07.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/4160350115' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2cf149e0-4c2f-46eb-9ca2-a51907aea3b0"}]: dispatch 2024-09-21T09:47:07.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2cf149e0-4c2f-46eb-9ca2-a51907aea3b0"}]: dispatch 2024-09-21T09:47:07.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2cf149e0-4c2f-46eb-9ca2-a51907aea3b0"}]': finished 2024-09-21T09:47:07.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[29191]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:47:07.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:07.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/4160350115' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2cf149e0-4c2f-46eb-9ca2-a51907aea3b0"}]: dispatch 2024-09-21T09:47:07.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2cf149e0-4c2f-46eb-9ca2-a51907aea3b0"}]: dispatch 2024-09-21T09:47:07.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2cf149e0-4c2f-46eb-9ca2-a51907aea3b0"}]': finished 2024-09-21T09:47:07.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[40061]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:47:07.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:06 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:07.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:06 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/4160350115' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2cf149e0-4c2f-46eb-9ca2-a51907aea3b0"}]: dispatch 2024-09-21T09:47:07.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:06 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2cf149e0-4c2f-46eb-9ca2-a51907aea3b0"}]: dispatch 2024-09-21T09:47:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:06 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2cf149e0-4c2f-46eb-9ca2-a51907aea3b0"}]': finished 2024-09-21T09:47:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:06 smithi045 ceph-mon[33087]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:47:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:06 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:08.157 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:07 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/4201305941' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:47:08.157 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:07 smithi032 ceph-mon[29191]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:08.158 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:07 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/4201305941' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:47:08.158 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:07 smithi032 ceph-mon[40061]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:07 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/4201305941' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:47:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:07 smithi045 ceph-mon[33087]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:10.068 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:09 smithi045 ceph-mon[33087]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:10.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:09 smithi032 ceph-mon[29191]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:09 smithi032 ceph-mon[40061]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:12.010 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:11 smithi045 ceph-mon[33087]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:11 smithi032 ceph-mon[29191]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:11 smithi032 ceph-mon[40061]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:12 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:47:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:12 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:12 smithi032 ceph-mon[29191]: Deploying daemon osd.4 on smithi045 2024-09-21T09:47:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:12 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:47:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:12 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:12 smithi032 ceph-mon[40061]: Deploying daemon osd.4 on smithi045 2024-09-21T09:47:13.217 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:12 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:47:13.217 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:12 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:13.217 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:12 smithi045 ceph-mon[33087]: Deploying daemon osd.4 on smithi045 2024-09-21T09:47:14.193 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:13 smithi045 ceph-mon[33087]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:14.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:13 smithi032 ceph-mon[29191]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:13 smithi032 ceph-mon[40061]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:15 smithi045 ceph-mon[33087]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:16.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:15 smithi032 ceph-mon[29191]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:15 smithi032 ceph-mon[40061]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:17.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:17 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:17.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:17 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:17.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:17 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:17.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:17 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:17 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:17 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:17 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:17 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:17 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:18.833 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:18 smithi045 ceph-mon[33087]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:18.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:18 smithi032 ceph-mon[29191]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:18.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:18 smithi032 ceph-mon[40061]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:20.777 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:20.777 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:20.777 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:20 smithi045 ceph-mon[33087]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:20.777 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:20.777 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:20.777 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:20 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:20.778 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:20 smithi045 ceph-mon[33087]: from='osd.4 [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:47:20.778 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:20 smithi045 ceph-mon[33087]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:47:20.813 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 4 on host 'smithi045' 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[29191]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[29191]: from='osd.4 [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[29191]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[40061]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[40061]: from='osd.4 [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:47:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:20 smithi032 ceph-mon[40061]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:47:21.452 DEBUG:teuthology.orchestra.run.smithi045:osd.4> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.4.service 2024-09-21T09:47:21.454 INFO:tasks.cephadm:Deploying osd.5 on smithi045 with /dev/vg_nvme/lv_3... 2024-09-21T09:47:21.454 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:47:21.734 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:21.734 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:21.734 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:21.734 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:21 smithi045 ceph-mon[33087]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:47:21.734 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:21 smithi045 ceph-mon[33087]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:47:21.734 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:21 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:21.734 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:21 smithi045 ceph-mon[33087]: from='osd.4 [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:21.735 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:21 smithi045 ceph-mon[33087]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:21.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[29191]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[29191]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[29191]: from='osd.4 [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[29191]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[40061]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:47:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[40061]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:47:21.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:21.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[40061]: from='osd.4 [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:21.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:21 smithi032 ceph-mon[40061]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:22.191 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:47:22.596 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 09:47:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T09:47:22.343+0000 7ff185ca5640 -1 osd.4 0 waiting for initial osdmap 2024-09-21T09:47:22.596 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 09:47:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T09:47:22.347+0000 7ff17d2af640 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:47:22.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[29191]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[29191]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[29191]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[29191]: from='osd.4 ' entity='osd.4' 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[40061]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[40061]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[40061]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:22.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:22 smithi032 ceph-mon[40061]: from='osd.4 ' entity='osd.4' 2024-09-21T09:47:23.031 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:22 smithi045 ceph-mon[33087]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-09-21T09:47:23.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:22 smithi045 ceph-mon[33087]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:47:23.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:22 smithi045 ceph-mon[33087]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:47:23.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:22 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:23.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:22 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:23.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:22 smithi045 ceph-mon[33087]: from='osd.4 ' entity='osd.4' 2024-09-21T09:47:24.308 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:24 smithi045 ceph-mon[33087]: purged_snaps scrub starts 2024-09-21T09:47:24.308 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:24 smithi045 ceph-mon[33087]: purged_snaps scrub ok 2024-09-21T09:47:24.308 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:24 smithi045 ceph-mon[33087]: osd.4 [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252] boot 2024-09-21T09:47:24.308 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:24 smithi045 ceph-mon[33087]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:47:24.308 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:24 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:24.308 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:24 smithi045 ceph-mon[33087]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:24.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[40061]: purged_snaps scrub starts 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[40061]: purged_snaps scrub ok 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[40061]: osd.4 [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252] boot 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[40061]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[40061]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[29191]: purged_snaps scrub starts 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[29191]: purged_snaps scrub ok 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[29191]: osd.4 [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252] boot 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[29191]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:47:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:24 smithi032 ceph-mon[29191]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:24.928 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:47:24.950 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi045:vg_nvme/lv_3 2024-09-21T09:47:25.265 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:25 smithi045 ceph-mon[33087]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:47:25.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:25 smithi032 ceph-mon[29191]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:47:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:25 smithi032 ceph-mon[40061]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:47:25.518 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:47:26.215 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:26 smithi045 ceph-mon[33087]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:47:26.215 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:26 smithi045 ceph-mon[33087]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:26.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:26 smithi032 ceph-mon[29191]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:47:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:26 smithi032 ceph-mon[29191]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:26 smithi032 ceph-mon[40061]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:47:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:26 smithi032 ceph-mon[40061]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:27.358 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:27 smithi032 ceph-mon[40061]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T09:47:27.358 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:27 smithi032 ceph-mon[29191]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T09:47:27.454 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:27 smithi045 ceph-mon[33087]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: Detected new or changed devices on smithi045 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi045 to 17092M 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.391 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-21T09:47:28.392 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:47:28.392 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:47:28.392 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:28 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: Detected new or changed devices on smithi045 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi045 to 17092M 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-21T09:47:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: Detected new or changed devices on smithi045 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi045 to 17092M 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-21T09:47:28.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:47:28.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:47:28.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:28 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:29.485 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:29 smithi045 ceph-mon[33087]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:29.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:29 smithi032 ceph-mon[29191]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:29.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:29 smithi032 ceph-mon[40061]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:30.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[29191]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/288589194' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6c78855-6756-4b0b-9a9c-8d102079540b"}]: dispatch 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6c78855-6756-4b0b-9a9c-8d102079540b"}]: dispatch 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a6c78855-6756-4b0b-9a9c-8d102079540b"}]': finished 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[29191]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[40061]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/288589194' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6c78855-6756-4b0b-9a9c-8d102079540b"}]: dispatch 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6c78855-6756-4b0b-9a9c-8d102079540b"}]: dispatch 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a6c78855-6756-4b0b-9a9c-8d102079540b"}]': finished 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[40061]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:47:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:30 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:30.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:30 smithi045 ceph-mon[33087]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-09-21T09:47:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:30 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/288589194' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6c78855-6756-4b0b-9a9c-8d102079540b"}]: dispatch 2024-09-21T09:47:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:30 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a6c78855-6756-4b0b-9a9c-8d102079540b"}]: dispatch 2024-09-21T09:47:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:30 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a6c78855-6756-4b0b-9a9c-8d102079540b"}]': finished 2024-09-21T09:47:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:30 smithi045 ceph-mon[33087]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:47:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:30 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:31.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:31 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/1819257161' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:47:31.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:31 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/1819257161' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:47:31.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:31 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/1819257161' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:47:32.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:32 smithi032 ceph-mon[29191]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:47:32.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:32 smithi032 ceph-mon[40061]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:47:32.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:32 smithi045 ceph-mon[33087]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:47:34.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:33 smithi045 ceph-mon[33087]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-21T09:47:34.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:33 smithi032 ceph-mon[29191]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-21T09:47:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:33 smithi032 ceph-mon[40061]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-21T09:47:36.023 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:35 smithi045 ceph-mon[33087]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:36.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:35 smithi032 ceph-mon[29191]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:35 smithi032 ceph-mon[40061]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:37.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:47:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:36 smithi032 ceph-mon[29191]: Deploying daemon osd.5 on smithi045 2024-09-21T09:47:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:47:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:36 smithi032 ceph-mon[40061]: Deploying daemon osd.5 on smithi045 2024-09-21T09:47:37.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:47:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:36 smithi045 ceph-mon[33087]: Deploying daemon osd.5 on smithi045 2024-09-21T09:47:38.183 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:37 smithi045 ceph-mon[33087]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:38.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:37 smithi032 ceph-mon[29191]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:37 smithi032 ceph-mon[40061]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:40.186 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:39 smithi045 ceph-mon[33087]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:40.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:39 smithi032 ceph-mon[29191]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:39 smithi032 ceph-mon[40061]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:41.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:41 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:41 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:41 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:41 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:41 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:41 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:41.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:41 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:41 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:41 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:42.761 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:42 smithi045 ceph-mon[33087]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:42.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:42 smithi032 ceph-mon[29191]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:42 smithi032 ceph-mon[40061]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:44.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[29191]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[40061]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:43 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:43 smithi045 ceph-mon[33087]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:43 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:43 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:43 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:43 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:43 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:44.868 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 5 on host 'smithi045' 2024-09-21T09:47:45.168 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:44 smithi045 ceph-mon[33087]: from='osd.5 [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:47:45.168 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:44 smithi045 ceph-mon[33087]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:47:45.168 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:44 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:45.168 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:44 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:45.168 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:44 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:45.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[29191]: from='osd.5 [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:47:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[29191]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:47:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[40061]: from='osd.5 [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:47:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[40061]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:47:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:44 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:45.451 DEBUG:teuthology.orchestra.run.smithi045:osd.5> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.5.service 2024-09-21T09:47:45.453 INFO:tasks.cephadm:Deploying osd.6 on smithi045 with /dev/vg_nvme/lv_2... 2024-09-21T09:47:45.453 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:47:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:45 smithi045 ceph-mon[33087]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:47:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:45 smithi045 ceph-mon[33087]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T09:47:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:45 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:45 smithi045 ceph-mon[33087]: from='osd.5 [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:45 smithi045 ceph-mon[33087]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:45 smithi045 ceph-mon[33087]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:46.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 09:47:45 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[43716]: 2024-09-21T09:47:45.938+0000 7f9bb63c3640 -1 osd.5 0 waiting for initial osdmap 2024-09-21T09:47:46.036 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 09:47:45 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[43716]: 2024-09-21T09:47:45.943+0000 7f9bb21ec640 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:47:46.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[29191]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[29191]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[29191]: from='osd.5 [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[29191]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[29191]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[40061]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[40061]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[40061]: from='osd.5 [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[40061]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:47:46.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:45 smithi032 ceph-mon[40061]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:47:47.012 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:47:47.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:46 smithi045 ceph-mon[33087]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:47:47.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:46 smithi045 ceph-mon[33087]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T09:47:47.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:46 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:47.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:46 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:46 smithi045 ceph-mon[33087]: osd.5 [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729] boot 2024-09-21T09:47:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:46 smithi045 ceph-mon[33087]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:47:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:46 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[29191]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:47:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[29191]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T09:47:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[29191]: osd.5 [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729] boot 2024-09-21T09:47:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[29191]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:47:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[40061]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:47:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[40061]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T09:47:47.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:47.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:47.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[40061]: osd.5 [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729] boot 2024-09-21T09:47:47.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[40061]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:47:47.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:46 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:47:48.194 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:47 smithi045 ceph-mon[33087]: purged_snaps scrub starts 2024-09-21T09:47:48.194 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:47 smithi045 ceph-mon[33087]: purged_snaps scrub ok 2024-09-21T09:47:48.194 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:47 smithi045 ceph-mon[33087]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:48.195 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:47 smithi045 ceph-mon[33087]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T09:47:48.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:47 smithi032 ceph-mon[29191]: purged_snaps scrub starts 2024-09-21T09:47:48.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:47 smithi032 ceph-mon[29191]: purged_snaps scrub ok 2024-09-21T09:47:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:47 smithi032 ceph-mon[29191]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:47 smithi032 ceph-mon[29191]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T09:47:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:47 smithi032 ceph-mon[40061]: purged_snaps scrub starts 2024-09-21T09:47:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:47 smithi032 ceph-mon[40061]: purged_snaps scrub ok 2024-09-21T09:47:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:47 smithi032 ceph-mon[40061]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:47 smithi032 ceph-mon[40061]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: Detected new or changed devices on smithi045 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi045 to 8546M 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:49 smithi045 ceph-mon[33087]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T09:47:50.130 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:47:50.149 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi045:vg_nvme/lv_2 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: Detected new or changed devices on smithi045 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi045 to 8546M 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[29191]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: Detected new or changed devices on smithi045 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi045 to 8546M 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:50.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:47:50.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:49 smithi032 ceph-mon[40061]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T09:47:50.351 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:47:51.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:50 smithi045 ceph-mon[33087]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:51.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:50 smithi045 ceph-mon[33087]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T09:47:51.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:50 smithi032 ceph-mon[29191]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:50 smithi032 ceph-mon[29191]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T09:47:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:50 smithi032 ceph-mon[40061]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:50 smithi032 ceph-mon[40061]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T09:47:52.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:51 smithi045 ceph-mon[33087]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:47:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:51 smithi045 ceph-mon[33087]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T09:47:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:51 smithi032 ceph-mon[29191]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:47:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:51 smithi032 ceph-mon[29191]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T09:47:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:51 smithi032 ceph-mon[40061]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:47:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:51 smithi032 ceph-mon[40061]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T09:47:53.133 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:52 smithi045 ceph-mon[33087]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:53.133 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:52 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:47:53.133 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:52 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:47:53.133 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:52 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:53.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:52 smithi032 ceph-mon[29191]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:52 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:47:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:52 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:47:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:52 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:52 smithi032 ceph-mon[40061]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:52 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:47:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:52 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:47:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:52 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:53 smithi045 ceph-mon[33087]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:54.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:53 smithi032 ceph-mon[29191]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:53 smithi032 ceph-mon[40061]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:54.975 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:54 smithi045 ceph-mon[33087]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:54.975 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:54 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/3162515215' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d476169d-dffc-4f1e-b60b-4f836f53285e"}]: dispatch 2024-09-21T09:47:54.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:54 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d476169d-dffc-4f1e-b60b-4f836f53285e"}]: dispatch 2024-09-21T09:47:54.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:54 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d476169d-dffc-4f1e-b60b-4f836f53285e"}]': finished 2024-09-21T09:47:54.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:54 smithi045 ceph-mon[33087]: osdmap e43: 7 total, 6 up, 7 in 2024-09-21T09:47:54.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:54 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:47:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[29191]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/3162515215' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d476169d-dffc-4f1e-b60b-4f836f53285e"}]: dispatch 2024-09-21T09:47:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d476169d-dffc-4f1e-b60b-4f836f53285e"}]: dispatch 2024-09-21T09:47:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d476169d-dffc-4f1e-b60b-4f836f53285e"}]': finished 2024-09-21T09:47:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[29191]: osdmap e43: 7 total, 6 up, 7 in 2024-09-21T09:47:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:47:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[40061]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:47:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/3162515215' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d476169d-dffc-4f1e-b60b-4f836f53285e"}]: dispatch 2024-09-21T09:47:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d476169d-dffc-4f1e-b60b-4f836f53285e"}]: dispatch 2024-09-21T09:47:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d476169d-dffc-4f1e-b60b-4f836f53285e"}]': finished 2024-09-21T09:47:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[40061]: osdmap e43: 7 total, 6 up, 7 in 2024-09-21T09:47:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:54 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:47:56.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:55 smithi045 ceph-mon[33087]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:47:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:55 smithi045 ceph-mon[33087]: Cluster is now healthy 2024-09-21T09:47:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:55 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/293382891' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:47:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:55 smithi032 ceph-mon[29191]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:47:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:55 smithi032 ceph-mon[29191]: Cluster is now healthy 2024-09-21T09:47:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:55 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/293382891' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:47:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:55 smithi032 ceph-mon[40061]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:47:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:55 smithi032 ceph-mon[40061]: Cluster is now healthy 2024-09-21T09:47:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:55 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/293382891' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:47:57.108 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:56 smithi032 ceph-mon[29191]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:47:57.108 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:56 smithi032 ceph-mon[40061]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:47:57.113 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:56 smithi045 ceph-mon[33087]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:47:58.193 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:57 smithi045 ceph-mon[33087]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-21T09:47:58.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:57 smithi032 ceph-mon[29191]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-21T09:47:58.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:57 smithi032 ceph-mon[40061]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-21T09:48:00.014 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:47:59 smithi045 ceph-mon[33087]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T09:48:00.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:47:59 smithi032 ceph-mon[29191]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T09:48:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:47:59 smithi032 ceph-mon[40061]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T09:48:01.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:00 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:48:01.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:00 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:01.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:00 smithi045 ceph-mon[33087]: Deploying daemon osd.6 on smithi045 2024-09-21T09:48:01.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:48:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:00 smithi032 ceph-mon[29191]: Deploying daemon osd.6 on smithi045 2024-09-21T09:48:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:00 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:48:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:00 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:00 smithi032 ceph-mon[40061]: Deploying daemon osd.6 on smithi045 2024-09-21T09:48:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:01 smithi032 ceph-mon[29191]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:48:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:01 smithi032 ceph-mon[40061]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:48:02.229 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:01 smithi045 ceph-mon[33087]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:48:04.163 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:03 smithi045 ceph-mon[33087]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:04.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:03 smithi032 ceph-mon[29191]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:03 smithi032 ceph-mon[40061]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:05.687 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:05 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:05.687 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:05 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:05.687 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:05 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:05.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:05 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:05 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:05 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:05 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:05 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:05 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:06.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:06 smithi045 ceph-mon[33087]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:06.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:06 smithi032 ceph-mon[29191]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:06.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:06 smithi032 ceph-mon[40061]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:08.509 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 6 on host 'smithi045' 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='osd.6 [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:08 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.098 DEBUG:teuthology.orchestra.run.smithi045:osd.6> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.6.service 2024-09-21T09:48:09.101 INFO:tasks.cephadm:Deploying osd.7 on smithi045 with /dev/vg_nvme/lv_1... 2024-09-21T09:48:09.101 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:48:09.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='osd.6 [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:09.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:09.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:09.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='osd.6 [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:48:09.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:48:09.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:09.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:09.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:08 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:10.016 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:09 smithi045 ceph-mon[33087]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:48:10.016 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:09 smithi045 ceph-mon[33087]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T09:48:10.016 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:09 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:10.017 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:09 smithi045 ceph-mon[33087]: from='osd.6 [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:10.017 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:09 smithi045 ceph-mon[33087]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:10.017 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 09:48:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T09:48:09.839+0000 7fb8d159f640 -1 osd.6 0 waiting for initial osdmap 2024-09-21T09:48:10.017 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 09:48:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T09:48:09.845+0000 7fb8cd3c8640 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:48:10.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[29191]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:48:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[29191]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T09:48:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[29191]: from='osd.6 [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[29191]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[40061]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:48:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[40061]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T09:48:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[40061]: from='osd.6 [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:09 smithi032 ceph-mon[40061]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:10.297 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:10 smithi045 ceph-mon[33087]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:48:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:10 smithi045 ceph-mon[33087]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T09:48:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:10 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:10 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:10 smithi045 ceph-mon[33087]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:11.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[29191]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:48:11.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[29191]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T09:48:11.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:11.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:11.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[29191]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:11.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[40061]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:48:11.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[40061]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T09:48:11.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:11.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:11.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:10 smithi032 ceph-mon[40061]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:48:12.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:11 smithi045 ceph-mon[33087]: purged_snaps scrub starts 2024-09-21T09:48:12.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:11 smithi045 ceph-mon[33087]: purged_snaps scrub ok 2024-09-21T09:48:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:11 smithi045 ceph-mon[33087]: osd.6 [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679] boot 2024-09-21T09:48:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:11 smithi045 ceph-mon[33087]: osdmap e46: 7 total, 7 up, 7 in 2024-09-21T09:48:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:11 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:11 smithi045 ceph-mon[33087]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T09:48:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[29191]: purged_snaps scrub starts 2024-09-21T09:48:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[29191]: purged_snaps scrub ok 2024-09-21T09:48:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[29191]: osd.6 [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679] boot 2024-09-21T09:48:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[29191]: osdmap e46: 7 total, 7 up, 7 in 2024-09-21T09:48:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[29191]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T09:48:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[40061]: purged_snaps scrub starts 2024-09-21T09:48:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[40061]: purged_snaps scrub ok 2024-09-21T09:48:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[40061]: osd.6 [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679] boot 2024-09-21T09:48:12.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[40061]: osdmap e46: 7 total, 7 up, 7 in 2024-09-21T09:48:12.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:12.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:11 smithi032 ceph-mon[40061]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T09:48:13.161 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:12 smithi045 ceph-mon[33087]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:13.162 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:12 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:13.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:12 smithi032 ceph-mon[40061]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:12 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:12 smithi032 ceph-mon[29191]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:12 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: Detected new or changed devices on smithi045 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi045 to 5697M 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: Detected new or changed devices on smithi045 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi045 to 5697M 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:13 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: Detected new or changed devices on smithi045 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi045 to 5697M 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:14.228 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:13 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:14.231 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-21T09:48:14.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi045:vg_nvme/lv_1 2024-09-21T09:48:14.459 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:15.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:14 smithi032 ceph-mon[29191]: pgmap v150: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2024-09-21T09:48:15.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:14 smithi032 ceph-mon[29191]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T09:48:15.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:14 smithi032 ceph-mon[40061]: pgmap v150: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2024-09-21T09:48:15.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:14 smithi032 ceph-mon[40061]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T09:48:15.250 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:14 smithi045 ceph-mon[33087]: pgmap v150: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2024-09-21T09:48:15.251 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:14 smithi045 ceph-mon[33087]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T09:48:16.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:15 smithi032 ceph-mon[29191]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T09:48:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:15 smithi032 ceph-mon[40061]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T09:48:16.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:15 smithi045 ceph-mon[33087]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T09:48:16.922 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:16 smithi045 ceph-mon[33087]: pgmap v153: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2024-09-21T09:48:16.923 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:16 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:48:16.923 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:16 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:48:16.923 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:16 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:17.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:16 smithi032 ceph-mon[29191]: pgmap v153: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2024-09-21T09:48:17.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:16 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:48:17.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:16 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:48:17.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:16 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:17.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:16 smithi032 ceph-mon[40061]: pgmap v153: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2024-09-21T09:48:17.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:16 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:48:17.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:16 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:48:17.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:16 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:18.020 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:17 smithi045 ceph-mon[33087]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:18.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:17 smithi032 ceph-mon[40061]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:18.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:17 smithi032 ceph-mon[29191]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:19.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[29191]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/1735645789' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "021a4fd0-f770-4799-9683-b0aa7cd593bc"}]: dispatch 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "021a4fd0-f770-4799-9683-b0aa7cd593bc"}]: dispatch 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[29191]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "021a4fd0-f770-4799-9683-b0aa7cd593bc"}]': finished 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[29191]: osdmap e51: 8 total, 7 up, 8 in 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[40061]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/1735645789' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "021a4fd0-f770-4799-9683-b0aa7cd593bc"}]: dispatch 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "021a4fd0-f770-4799-9683-b0aa7cd593bc"}]: dispatch 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[40061]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "021a4fd0-f770-4799-9683-b0aa7cd593bc"}]': finished 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[40061]: osdmap e51: 8 total, 7 up, 8 in 2024-09-21T09:48:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:18 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:19.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:18 smithi045 ceph-mon[33087]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:48:19.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:18 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/1735645789' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "021a4fd0-f770-4799-9683-b0aa7cd593bc"}]: dispatch 2024-09-21T09:48:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:18 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "021a4fd0-f770-4799-9683-b0aa7cd593bc"}]: dispatch 2024-09-21T09:48:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:18 smithi045 ceph-mon[33087]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "021a4fd0-f770-4799-9683-b0aa7cd593bc"}]': finished 2024-09-21T09:48:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:18 smithi045 ceph-mon[33087]: osdmap e51: 8 total, 7 up, 8 in 2024-09-21T09:48:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:18 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:20.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:19 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/2155302705' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:48:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:19 smithi032 ceph-mon[29191]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:20.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:19 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/2155302705' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:48:20.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:19 smithi032 ceph-mon[40061]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:20.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:19 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/2155302705' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:48:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:19 smithi045 ceph-mon[33087]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:22.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:21 smithi032 ceph-mon[29191]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:22.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:21 smithi032 ceph-mon[40061]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:22.217 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:21 smithi045 ceph-mon[33087]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:24.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:23 smithi032 ceph-mon[29191]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:23 smithi032 ceph-mon[40061]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:24.237 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:23 smithi045 ceph-mon[33087]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:24 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:48:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:24 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:24 smithi032 ceph-mon[40061]: Deploying daemon osd.7 on smithi045 2024-09-21T09:48:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:24 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:48:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:24 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:24 smithi032 ceph-mon[29191]: Deploying daemon osd.7 on smithi045 2024-09-21T09:48:25.276 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:24 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:48:25.276 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:24 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:25.277 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:24 smithi045 ceph-mon[33087]: Deploying daemon osd.7 on smithi045 2024-09-21T09:48:26.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:25 smithi045 ceph-mon[33087]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:26.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:25 smithi032 ceph-mon[29191]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:25 smithi032 ceph-mon[40061]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:28.143 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:27 smithi045 ceph-mon[33087]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:28.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:27 smithi032 ceph-mon[40061]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:28.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:27 smithi032 ceph-mon[29191]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:29.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:29 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:29 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:29 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:29.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:29 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:29.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:29 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:29.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:29 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:29.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:29 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:30.642 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:30 smithi045 ceph-mon[33087]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:30.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:30 smithi032 ceph-mon[29191]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:30 smithi032 ceph-mon[40061]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:31.859 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 7 on host 'smithi045' 2024-09-21T09:48:32.511 DEBUG:teuthology.orchestra.run.smithi045:osd.7> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.7.service 2024-09-21T09:48:32.514 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-21T09:48:32.514 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T09:48:32.731 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:33.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:33.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:33.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[29191]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:33.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[29191]: from='osd.7 [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:48:33.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[29191]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:48:33.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:33.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:33.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[40061]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:33.181 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[40061]: from='osd.7 [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:48:33.181 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:32 smithi032 ceph-mon[40061]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:48:33.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:32 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:33.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:32 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:33.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:32 smithi045 ceph-mon[33087]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:32 smithi045 ceph-mon[33087]: from='osd.7 [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:48:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:32 smithi045 ceph-mon[33087]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:48:33.957 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:33 smithi045 ceph-mon[33087]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:48:33.957 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:33 smithi045 ceph-mon[33087]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T09:48:33.958 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:33 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:33.958 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:33 smithi045 ceph-mon[33087]: from='osd.7 [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:33.958 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:33 smithi045 ceph-mon[33087]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:33.958 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:33 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:33.958 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:33 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:33.958 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 09:48:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T09:48:33.867+0000 7f7b677f0640 -1 osd.7 0 waiting for initial osdmap 2024-09-21T09:48:33.958 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 09:48:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T09:48:33.875+0000 7f7b63619640 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:48:34.139 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[40061]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:48:34.140 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[40061]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T09:48:34.140 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:34.140 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[40061]: from='osd.7 [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:34.140 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[40061]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:34.140 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:34.140 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:34.140 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[29191]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:48:34.140 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[29191]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T09:48:34.140 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:34.141 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[29191]: from='osd.7 [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:34.141 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[29191]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-21T09:48:34.141 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:34.141 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:33 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:34.824 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:48:35.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[29191]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:48:35.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[29191]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[29191]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3460769885' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[40061]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[40061]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[40061]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:34 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3460769885' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:48:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:34 smithi045 ceph-mon[33087]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-21T09:48:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:34 smithi045 ceph-mon[33087]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T09:48:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:34 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:34 smithi045 ceph-mon[33087]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:34 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:34 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3460769885' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:48:35.480 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1726912090,"num_in_osds":8,"osd_in_since":1726912098,"num_remapped_pgs":0} 2024-09-21T09:48:36.142 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:35 smithi045 ceph-mon[33087]: purged_snaps scrub starts 2024-09-21T09:48:36.142 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:35 smithi045 ceph-mon[33087]: purged_snaps scrub ok 2024-09-21T09:48:36.142 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:35 smithi045 ceph-mon[33087]: osd.7 [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350] boot 2024-09-21T09:48:36.142 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:35 smithi045 ceph-mon[33087]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:48:36.142 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:35 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:36.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[29191]: purged_snaps scrub starts 2024-09-21T09:48:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[29191]: purged_snaps scrub ok 2024-09-21T09:48:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[29191]: osd.7 [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350] boot 2024-09-21T09:48:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[29191]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:48:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[40061]: purged_snaps scrub starts 2024-09-21T09:48:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[40061]: purged_snaps scrub ok 2024-09-21T09:48:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[40061]: osd.7 [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350] boot 2024-09-21T09:48:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[40061]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:48:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:35 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:36.481 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T09:48:36.688 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:37.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:37.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:48:37.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.071 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:37.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:36 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:37.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:37.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:36 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:38.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:37 smithi032 ceph-mon[40061]: Detected new or changed devices on smithi045 2024-09-21T09:48:38.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:37 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi045 to 4273M 2024-09-21T09:48:38.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:37 smithi032 ceph-mon[40061]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:48:38.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:37 smithi032 ceph-mon[40061]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:48:38.110 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:37 smithi032 ceph-mon[29191]: Detected new or changed devices on smithi045 2024-09-21T09:48:38.110 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:37 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi045 to 4273M 2024-09-21T09:48:38.110 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:37 smithi032 ceph-mon[29191]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:48:38.110 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:37 smithi032 ceph-mon[29191]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:48:38.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:37 smithi045 ceph-mon[33087]: Detected new or changed devices on smithi045 2024-09-21T09:48:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:37 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi045 to 4273M 2024-09-21T09:48:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:37 smithi045 ceph-mon[33087]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:48:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:37 smithi045 ceph-mon[33087]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:48:38.490 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:48:39.123 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:38 smithi032 ceph-mon[29191]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:39.123 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:38 smithi032 ceph-mon[29191]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:48:39.123 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:38 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1789066087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:48:39.123 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:38 smithi032 ceph-mon[40061]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:39.123 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:38 smithi032 ceph-mon[40061]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:48:39.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:38 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1789066087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:48:39.146 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912114,"num_in_osds":8,"osd_in_since":1726912098,"num_remapped_pgs":0} 2024-09-21T09:48:39.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:48:39.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:38 smithi045 ceph-mon[33087]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:39.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:38 smithi045 ceph-mon[33087]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:48:39.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:38 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1789066087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:48:39.352 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:40.046 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:39 smithi032 ceph-mon[40061]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:40.046 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:39 smithi032 ceph-mon[29191]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:40.186 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:39 smithi045 ceph-mon[33087]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:41.017 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:48:41.018 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":58,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:43:37.317307+0000","modified":"2024-09-21T09:48:37.925232+0000","last_up_change":"2024-09-21T09:48:34.864213+0000","last_in_change":"2024-09-21T09:48:18.335792+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:46:40.328691+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0f6f5dda-c134-4283-a91d-cf501ef19062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6803","nonce":2765215923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6805","nonce":2765215923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6809","nonce":2765215923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6807","nonce":2765215923}]},"public_addr":"172.21.15.32:6803/2765215923","cluster_addr":"172.21.15.32:6805/2765215923","heartbeat_back_addr":"172.21.15.32:6809/2765215923","heartbeat_front_addr":"172.21.15.32:6807/2765215923","state":["exists","up"]},{"osd":1,"uuid":"bdaa43fd-b969-4d06-b6eb-97b415c4e118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6811","nonce":3808018383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6813","nonce":3808018383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6817","nonce":3808018383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6815","nonce":3808018383}]},"public_addr":"172.21.15.32:6811/3808018383","cluster_addr":"172.21.15.32:6813/3808018383","heartbeat_back_addr":"172.21.15.32:6817/3808018383","heartbeat_front_addr":"172.21.15.32:6815/3808018383","state":["exists","up"]},{"osd":2,"uuid":"1f12d9f2-90b0-44f6-8432-0b398194b3b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6819","nonce":2468805298}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6821","nonce":2468805298}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6825","nonce":2468805298}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6823","nonce":2468805298}]},"public_addr":"172.21.15.32:6819/2468805298","cluster_addr":"172.21.15.32:6821/2468805298","heartbeat_back_addr":"172.21.15.32:6825/2468805298","heartbeat_front_addr":"172.21.15.32:6823/2468805298","state":["exists","up"]},{"osd":3,"uuid":"6a568fd5-5fd9-46f8-b3af-9fa196ed617b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6827","nonce":2765373470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6829","nonce":2765373470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6833","nonce":2765373470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6831","nonce":2765373470}]},"public_addr":"172.21.15.32:6827/2765373470","cluster_addr":"172.21.15.32:6829/2765373470","heartbeat_back_addr":"172.21.15.32:6833/2765373470","heartbeat_front_addr":"172.21.15.32:6831/2765373470","state":["exists","up"]},{"osd":4,"uuid":"2cf149e0-4c2f-46eb-9ca2-a51907aea3b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6801","nonce":413300252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6803","nonce":413300252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6807","nonce":413300252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6805","nonce":413300252}]},"public_addr":"172.21.15.45:6801/413300252","cluster_addr":"172.21.15.45:6803/413300252","heartbeat_back_addr":"172.21.15.45:6807/413300252","heartbeat_front_addr":"172.21.15.45:6805/413300252","state":["exists","up"]},{"osd":5,"uuid":"a6c78855-6756-4b0b-9a9c-8d102079540b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6809","nonce":3168849729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6811","nonce":3168849729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6815","nonce":3168849729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6813","nonce":3168849729}]},"public_addr":"172.21.15.45:6809/3168849729","cluster_addr":"172.21.15.45:6811/3168849729","heartbeat_back_addr":"172.21.15.45:6815/3168849729","heartbeat_front_addr":"172.21.15.45:6813/3168849729","state":["exists","up"]},{"osd":6,"uuid":"d476169d-dffc-4f1e-b60b-4f836f53285e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6817","nonce":2158139679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6819","nonce":2158139679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6823","nonce":2158139679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6821","nonce":2158139679}]},"public_addr":"172.21.15.45:6817/2158139679","cluster_addr":"172.21.15.45:6819/2158139679","heartbeat_back_addr":"172.21.15.45:6823/2158139679","heartbeat_front_addr":"172.21.15.45:6821/2158139679","state":["exists","up"]},{"osd":7,"uuid":"021a4fd0-f770-4799-9683-b0aa7cd593bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6825","nonce":3336053350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6827","nonce":3336053350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6831","nonce":3336053350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6829","nonce":3336053350}]},"public_addr":"172.21.15.45:6825/3336053350","cluster_addr":"172.21.15.45:6827/3336053350","heartbeat_back_addr":"172.21.15.45:6831/3336053350","heartbeat_front_addr":"172.21.15.45:6829/3336053350","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:50.697928+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:13.831263+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:37.360357+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:00.010474+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:21.607129+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:45.583237+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:09.486296+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:33.456690+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/2042031504":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6801/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2895073807":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6801/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/533058733":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/1178553739":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/3332221400":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/230160407":"2024-09-22T09:43:58.900171+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:48:41.372 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:41 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/269130843' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:48:41.372 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:41 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/269130843' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:48:41.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:41 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/269130843' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:48:41.666 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-21T09:46:40.328691+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-09-21T09:48:41.666 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-21T09:48:41.879 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:42.342 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:42 smithi032 ceph-mon[29191]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:48:42.342 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:42 smithi032 ceph-mon[40061]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:48:42.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:42 smithi045 ceph-mon[33087]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:48:43.748 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-09-21T09:48:44.068 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:43 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3319792696' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:48:44.068 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:43 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3319792696' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:48:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:43 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3319792696' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:48:44.470 INFO:tasks.cephadm:Adding prometheus.a on smithi045 2024-09-21T09:48:44.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch apply prometheus '1;smithi045=a' 2024-09-21T09:48:44.690 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:45.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:44 smithi045 ceph-mon[33087]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-21T09:48:45.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:44 smithi032 ceph-mon[29191]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-21T09:48:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:44 smithi032 ceph-mon[40061]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-21T09:48:46.013 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:45 smithi045 ceph-mon[33087]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:48:46.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:45 smithi032 ceph-mon[29191]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:48:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:45 smithi032 ceph-mon[40061]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:48:46.538 INFO:teuthology.orchestra.run.smithi045.stdout:Scheduled prometheus update... 2024-09-21T09:48:47.259 DEBUG:teuthology.orchestra.run.smithi045:prometheus.a> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service 2024-09-21T09:48:47.262 INFO:tasks.cephadm:Adding node-exporter.a on smithi032 2024-09-21T09:48:47.262 INFO:tasks.cephadm:Adding node-exporter.b on smithi045 2024-09-21T09:48:47.262 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch apply node-exporter '2;smithi032=a;smithi045=b' 2024-09-21T09:48:47.519 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:47.823 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:47 smithi045 ceph-mon[33087]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi045=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:47.824 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:47 smithi045 ceph-mon[33087]: Saving service prometheus spec with placement smithi045=a;count:1 2024-09-21T09:48:47.824 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:47.824 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:47.824 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:47.824 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:47.824 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:47 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:47.824 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:47 smithi045 ceph-mon[33087]: Deploying daemon prometheus.a on smithi045 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[29191]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi045=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[29191]: Saving service prometheus spec with placement smithi045=a;count:1 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[29191]: Deploying daemon prometheus.a on smithi045 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[40061]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi045=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[40061]: Saving service prometheus spec with placement smithi045=a;count:1 2024-09-21T09:48:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:47.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:47.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:47.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:47.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:47.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:47 smithi032 ceph-mon[40061]: Deploying daemon prometheus.a on smithi045 2024-09-21T09:48:48.754 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:48 smithi045 ceph-mon[33087]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:48:48.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:48 smithi032 ceph-mon[29191]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:48:48.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:48 smithi032 ceph-mon[40061]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:48:49.879 INFO:teuthology.orchestra.run.smithi045.stdout:Scheduled node-exporter update... 2024-09-21T09:48:50.635 DEBUG:teuthology.orchestra.run.smithi032:node-exporter.a> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.a.service 2024-09-21T09:48:50.638 DEBUG:teuthology.orchestra.run.smithi045:node-exporter.b> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.b.service 2024-09-21T09:48:50.641 INFO:tasks.cephadm:Adding alertmanager.a on smithi032 2024-09-21T09:48:50.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch apply alertmanager '1;smithi032=a' 2024-09-21T09:48:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:50 smithi032 ceph-mon[29191]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:48:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:50 smithi032 ceph-mon[29191]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi032=a;smithi045=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:50 smithi032 ceph-mon[29191]: Saving service node-exporter spec with placement smithi032=a;smithi045=b;count:2 2024-09-21T09:48:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:50 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:50 smithi032 ceph-mon[40061]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:48:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:50 smithi032 ceph-mon[40061]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi032=a;smithi045=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:50 smithi032 ceph-mon[40061]: Saving service node-exporter spec with placement smithi032=a;smithi045=b;count:2 2024-09-21T09:48:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:50 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:51.104 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:51.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:50 smithi045 ceph-mon[33087]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:48:51.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:50 smithi045 ceph-mon[33087]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi032=a;smithi045=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:51.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:50 smithi045 ceph-mon[33087]: Saving service node-exporter spec with placement smithi032=a;smithi045=b;count:2 2024-09-21T09:48:51.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:50 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:53.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:52 smithi032 ceph-mon[29191]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:48:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:52 smithi032 ceph-mon[40061]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:48:53.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:52 smithi045 ceph-mon[33087]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:48:54.206 INFO:teuthology.orchestra.run.smithi045.stdout:Scheduled alertmanager update... 2024-09-21T09:48:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:54 smithi032 ceph-mon[40061]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:54 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:54 smithi032 ceph-mon[29191]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:54 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:55.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:54 smithi045 ceph-mon[33087]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:54 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:48:55.474 DEBUG:teuthology.orchestra.run.smithi032:alertmanager.a> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@alertmanager.a.service 2024-09-21T09:48:55.477 INFO:tasks.cephadm:Adding grafana.a on smithi045 2024-09-21T09:48:55.477 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph orch apply grafana '1;smithi045=a' 2024-09-21T09:48:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:55 smithi032 ceph-mon[29191]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi032=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:55 smithi032 ceph-mon[29191]: Saving service alertmanager spec with placement smithi032=a;count:1 2024-09-21T09:48:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:55 smithi032 ceph-mon[40061]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi032=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:55 smithi032 ceph-mon[40061]: Saving service alertmanager spec with placement smithi032=a;count:1 2024-09-21T09:48:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:55 smithi045 ceph-mon[33087]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi032=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:55 smithi045 ceph-mon[33087]: Saving service alertmanager spec with placement smithi032=a;count:1 2024-09-21T09:48:56.316 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:56.978 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:56 smithi045 ceph-mon[33087]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:56 smithi032 ceph-mon[29191]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:56 smithi032 ceph-mon[40061]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:57 smithi032 ceph-mon[40061]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:57 smithi032 ceph-mon[29191]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:58.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:57 smithi045 ceph-mon[33087]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:59.939 INFO:teuthology.orchestra.run.smithi045.stdout:Scheduled grafana update... 2024-09-21T09:49:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:48:59 smithi032 ceph-mon[29191]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:48:59 smithi032 ceph-mon[40061]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:48:59 smithi045 ceph-mon[33087]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:00.751 DEBUG:teuthology.orchestra.run.smithi045:grafana.a> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@grafana.a.service 2024-09-21T09:49:00.754 INFO:tasks.cephadm:Setting up client nodes... 2024-09-21T09:49:00.754 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:49:00.973 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:01.029 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:00 smithi045 ceph-mon[33087]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi045=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:01.029 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:00 smithi045 ceph-mon[33087]: Saving service grafana spec with placement smithi045=a;count:1 2024-09-21T09:49:01.029 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:00 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:01.029 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:49:00 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:49:00.970Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:49:01.029 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:49:01 smithi045 bash[59479]: 6d18867ea721b1573a8facb1b3b0e80f3b40d36e62b8bad907b14e0bb30b2449 2024-09-21T09:49:01.070 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:00 smithi032 ceph-mon[40061]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi045=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:01.070 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:00 smithi032 ceph-mon[40061]: Saving service grafana spec with placement smithi045=a;count:1 2024-09-21T09:49:01.070 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:00 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:01.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:00 smithi032 ceph-mon[29191]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi045=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:01.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:00 smithi032 ceph-mon[29191]: Saving service grafana spec with placement smithi045=a;count:1 2024-09-21T09:49:01.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:00 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:01.284 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:49:01 smithi045 systemd[1]: Started Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:02.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:02.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:02.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:02.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:49:02.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[29191]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:02.428 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ignoring --setuser ceph since I am not root 2024-09-21T09:49:02.428 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ignoring --setgroup ceph since I am not root 2024-09-21T09:49:02.428 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:02.267+0000 7f809d932040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:49:02.428 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:02.327+0000 7f809d932040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:49:02.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:02.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:02.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:02.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:49:02.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-mon[40061]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:02.463 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:02.463 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:02.463 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' 2024-09-21T09:49:02.463 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:49:02.463 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-mon[33087]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:02.464 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: ignoring --setuser ceph since I am not root 2024-09-21T09:49:02.464 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: ignoring --setgroup ceph since I am not root 2024-09-21T09:49:02.464 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-mgr[34944]: -- 172.21.15.45:0/1022128367 <== mon.1 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556335d0ad00 con 0x556335cde800 2024-09-21T09:49:02.464 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:02.250+0000 7f0b94b34040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:49:02.464 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:02.308+0000 7f0b94b34040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:49:02.464 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:02.461+0000 7f0b94b34040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:49:02.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:02.486+0000 7f809d932040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:49:02.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:02.556+0000 7f809d932040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:49:02.784 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:02.529+0000 7f0b94b34040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:49:02.784 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:02.679+0000 7f0b94b34040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:49:02.880 INFO:teuthology.orchestra.run.smithi032.stdout:[client.0] 2024-09-21T09:49:02.880 INFO:teuthology.orchestra.run.smithi032.stdout: key = AQCOlu5m7u8yNBAAN85isiM51pklvNBMjoITTQ== 2024-09-21T09:49:03.100 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:02 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:02.823+0000 7f0b94b34040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:49:03.101 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:02.711+0000 7f809d932040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:49:03.101 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:02 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:02.858+0000 7f809d932040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:49:03.394 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:03.142+0000 7f0b94b34040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:49:03.395 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:03.204+0000 7f0b94b34040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:49:03.395 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:03.393+0000 7f0b94b34040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:49:03.395 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:03 smithi045 ceph-mon[33087]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:49:03.395 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:03 smithi045 ceph-mon[33087]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-21T09:49:03.395 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:03 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/4087370973' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:49:03.395 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:03 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/4087370973' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:49:03.424 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-mon[40061]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:49:03.425 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-mon[40061]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-21T09:49:03.425 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/4087370973' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:49:03.425 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/4087370973' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:49:03.425 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:03.186+0000 7f809d932040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:49:03.425 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:03.249+0000 7f809d932040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:49:03.425 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-mon[29191]: from='mgr.14150 172.21.15.32:0/2441179410' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:49:03.425 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-mon[29191]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-21T09:49:03.425 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/4087370973' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:49:03.425 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/4087370973' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:49:03.446 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:49:03.446 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-21T09:49:03.446 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-21T09:49:03.486 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:49:03.697 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:49:03.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:03.443+0000 7f809d932040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:49:04.393 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:04 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:04.322+0000 7f0b94b34040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:49:04.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:04 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:04.350+0000 7f809d932040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:49:04.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:04 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:04.418+0000 7f809d932040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:49:04.784 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:04 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:04.391+0000 7f0b94b34040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:49:05.415 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:05.141+0000 7f0b94b34040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:49:05.415 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:05.263+0000 7f0b94b34040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:49:05.428 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:05.159+0000 7f809d932040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:49:05.428 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:05.278+0000 7f809d932040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:49:05.564 INFO:teuthology.orchestra.run.smithi045.stdout:[client.1] 2024-09-21T09:49:05.564 INFO:teuthology.orchestra.run.smithi045.stdout: key = AQCRlu5mqalWIRAAF2ThNiMBi5SWaX+//Ehj2g== 2024-09-21T09:49:05.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:05.426+0000 7f809d932040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:49:05.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:05.489+0000 7f809d932040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:49:05.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:05.643+0000 7f809d932040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:49:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-mon[40061]: from='client.? 172.21.15.45:0/1630497828' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:49:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-mon[40061]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:49:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-mon[40061]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:49:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-mon[29191]: from='client.? 172.21.15.45:0/1630497828' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:49:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-mon[29191]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:49:05.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-mon[29191]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:49:05.708 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-mon[33087]: from='client.? 172.21.15.45:0/1630497828' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:49:05.708 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-mon[33087]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:49:05.708 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-mon[33087]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:49:05.708 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:05.413+0000 7f0b94b34040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:49:05.708 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:05.476+0000 7f0b94b34040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:49:05.708 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:05.635+0000 7f0b94b34040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:49:05.708 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:05.706+0000 7f0b94b34040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:49:05.982 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:05.768+0000 7f0b94b34040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:49:05.982 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:05 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:05.932+0000 7f0b94b34040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:49:06.007 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:05.713+0000 7f809d932040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:49:06.007 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:05.775+0000 7f809d932040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:49:06.007 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:05.937+0000 7f809d932040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:49:06.008 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:06 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:06.005+0000 7f809d932040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:49:06.177 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-21T09:49:06.177 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-21T09:49:06.178 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-21T09:49:06.218 INFO:tasks.cephadm:Creating RBD pool 2024-09-21T09:49:06.219 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create rbd 8 2024-09-21T09:49:06.284 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:06 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:06.000+0000 7f0b94b34040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:49:06.284 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:06 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:06.069+0000 7f0b94b34040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:49:06.284 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:06 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:06.154+0000 7f0b94b34040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:49:06.284 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:06 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:06.220+0000 7f0b94b34040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:49:06.284 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:06 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:06.282+0000 7f0b94b34040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:49:06.286 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:06 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:06.073+0000 7f809d932040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:49:06.286 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:06 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:06.159+0000 7f809d932040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:49:06.286 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:06 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:06.220+0000 7f809d932040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:49:06.426 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:06.547 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:06 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:06.283+0000 7f809d932040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:49:07.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:07.122+0000 7f809d932040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:49:07.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:49:07] ENGINE Bus STARTING 2024-09-21T09:49:07.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: CherryPy Checker: 2024-09-21T09:49:07.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: The Application mounted at '' has an empty config. 2024-09-21T09:49:07.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:07.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:49:07] ENGINE Serving on http://:::9283 2024-09-21T09:49:07.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:49:07] ENGINE Bus STARTED 2024-09-21T09:49:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: Standby manager daemon x restarted 2024-09-21T09:49:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: Standby manager daemon x started 2024-09-21T09:49:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:49:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:49:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:49:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:49:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: Active manager daemon y restarted 2024-09-21T09:49:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: Activating manager daemon y 2024-09-21T09:49:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: mgrmap e17: y(active, starting, since 0.00738937s), standbys: x 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:49:07.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[40061]: Manager daemon y is now available 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: Standby manager daemon x restarted 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: Standby manager daemon x started 2024-09-21T09:49:07.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: Active manager daemon y restarted 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: Activating manager daemon y 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: mgrmap e17: y(active, starting, since 0.00738937s), standbys: x 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:49:07.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:49:07.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:49:07.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:49:07.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:49:07.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:49:07.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:49:07.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:49:07.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:49:07.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:49:07.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:49:07.463 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:49:07.463 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:49:07.463 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:49:07.463 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:49:07.463 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:07 smithi032 ceph-mon[29191]: Manager daemon y is now available 2024-09-21T09:49:07.478 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:07.106+0000 7f0b94b34040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:49:07.478 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: [21/Sep/2024:09:49:07] ENGINE Bus STARTING 2024-09-21T09:49:07.478 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: CherryPy Checker: 2024-09-21T09:49:07.478 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: The Application mounted at '' has an empty config. 2024-09-21T09:49:07.478 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: 2024-09-21T09:49:07.478 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: [21/Sep/2024:09:49:07] ENGINE Serving on http://:::9283 2024-09-21T09:49:07.478 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[34940]: [21/Sep/2024:09:49:07] ENGINE Bus STARTED 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: Standby manager daemon x restarted 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: Standby manager daemon x started 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/2778181475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: Active manager daemon y restarted 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: Activating manager daemon y 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: mgrmap e17: y(active, starting, since 0.00738937s), standbys: x 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:49:07.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:49:07.480 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:07 smithi045 ceph-mon[33087]: Manager daemon y is now available 2024-09-21T09:49:08.379 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:08.379 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:08.379 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[40061]: mgrmap e18: y(active, since 1.01221s), standbys: x 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[40061]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:49:08.380 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[29191]: mgrmap e18: y(active, since 1.01221s), standbys: x 2024-09-21T09:49:08.381 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:08 smithi032 ceph-mon[29191]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:08.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:08 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:08 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:08 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:08 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:49:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:08 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:49:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:08 smithi045 ceph-mon[33087]: mgrmap e18: y(active, since 1.01221s), standbys: x 2024-09-21T09:49:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:08 smithi045 ceph-mon[33087]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:09.178 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'rbd' created 2024-09-21T09:49:09.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3839179348' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:49:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-mon[40061]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:49:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-mon[40061]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3839179348' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:49:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-mon[29191]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:49:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-mon[29191]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:09.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:09.297+0000 7f7ffb6ac640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:49:09] ENGINE Error in HTTPServer.serve 2024-09-21T09:49:09.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: Traceback (most recent call last): 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self._connections.run(self.expiration_interval) 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self._run(expiration_interval) 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: s = self.context.wrap_socket( 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: return self.sslsocket_class._create( 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self.do_handshake() 2024-09-21T09:49:09.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:49:09.459 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: self._sslobj.do_handshake() 2024-09-21T09:49:09.459 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:49:09.459 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:49:09.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:09 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3839179348' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:49:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:09 smithi045 ceph-mon[33087]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:49:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:09 smithi045 ceph-mon[33087]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:09.899 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- rbd pool init rbd 2024-09-21T09:49:10.115 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:10.178 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: [21/Sep/2024:09:49:09] ENGINE Bus STARTING 2024-09-21T09:49:10.178 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:49:10.178 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:49:10.178 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-21T09:49:10.178 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.178 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: [21/Sep/2024:09:49:09] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: [21/Sep/2024:09:49:09] ENGINE Error in HTTPServer.serve 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:49:10.179 INFO:journalctl@ceph.mon.c.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout: self.do_handshake() 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: [21/Sep/2024:09:49:09] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: [21/Sep/2024:09:49:09] ENGINE Bus STARTED 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.181 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.181 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.181 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi045 to 4017M 2024-09-21T09:49:10.181 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.426 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:49:10.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:49:10.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:49:10.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[40061]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:49:10.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: [21/Sep/2024:09:49:09] ENGINE Bus STARTING 2024-09-21T09:49:10.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:49:10.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:49:10.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: [21/Sep/2024:09:49:09] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: [21/Sep/2024:09:49:09] ENGINE Error in HTTPServer.serve 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:49:10.428 INFO:journalctl@ceph.mon.a.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout: self.do_handshake() 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: [21/Sep/2024:09:49:09] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: [21/Sep/2024:09:49:09] ENGINE Bus STARTED 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi045 to 4017M 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:49:10.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:10 smithi032 ceph-mon[29191]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:49:10.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: [21/Sep/2024:09:49:09] ENGINE Bus STARTING 2024-09-21T09:49:10.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:49:10.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:49:10.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-21T09:49:10.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: [21/Sep/2024:09:49:09] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: [21/Sep/2024:09:49:09] ENGINE Error in HTTPServer.serve 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: Traceback (most recent call last): 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._run(expiration_interval) 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: s = self.context.wrap_socket( 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:49:10.437 INFO:journalctl@ceph.mon.b.smithi045.stdout: return self.sslsocket_class._create( 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout: self.do_handshake() 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._sslobj.do_handshake() 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: [21/Sep/2024:09:49:09] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: [21/Sep/2024:09:49:09] ENGINE Bus STARTED 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.438 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:49:10.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi045 to 4017M 2024-09-21T09:49:10.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:10.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:49:10.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:49:10.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:49:10.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:10 smithi045 ceph-mon[33087]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:49:11.345 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:49:11.345 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:49:11.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:49:11.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:49:11.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:49:11.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:49:11.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:11.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:11.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:49:11.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[40061]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:11.346 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:49:11.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:49:11.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:49:11.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:49:11.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:49:11.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:49:11.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:11.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:11.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:49:11.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:11 smithi032 ceph-mon[29191]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:49:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:49:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:49:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:49:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:49:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:49:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:11.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:49:11.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:11 smithi045 ceph-mon[33087]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[29191]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[29191]: Deploying daemon alertmanager.a on smithi032 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/530324243' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[29191]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[29191]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[40061]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:12.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:12.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:12.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[40061]: Deploying daemon alertmanager.a on smithi032 2024-09-21T09:49:12.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/530324243' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:49:12.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[40061]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:12.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:12 smithi032 ceph-mon[40061]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-21T09:49:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:12 smithi045 ceph-mon[33087]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:49:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:12 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:12 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:12 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:12 smithi045 ceph-mon[33087]: Deploying daemon alertmanager.a on smithi032 2024-09-21T09:49:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:12 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/530324243' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:49:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:12 smithi045 ceph-mon[33087]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:12 smithi045 ceph-mon[33087]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-21T09:49:13.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:13 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/530324243' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:49:13.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:13 smithi032 ceph-mon[29191]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:49:13.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:13 smithi032 ceph-mon[29191]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:13.627 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:13 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/530324243' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:49:13.627 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:13 smithi032 ceph-mon[40061]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:49:13.627 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:13 smithi032 ceph-mon[40061]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:13 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/530324243' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:49:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:13 smithi045 ceph-mon[33087]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:49:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:13 smithi045 ceph-mon[33087]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:13.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:49:13] "GET /metrics HTTP/1.1" 200 23669 "" "Prometheus/2.43.0" 2024-09-21T09:49:14.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:14 smithi032 ceph-mon[40061]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:14.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:14 smithi032 ceph-mon[40061]: Cluster is now healthy 2024-09-21T09:49:14.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:14 smithi032 ceph-mon[40061]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:49:14.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:14 smithi032 ceph-mon[29191]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:14.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:14 smithi032 ceph-mon[29191]: Cluster is now healthy 2024-09-21T09:49:14.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:14 smithi032 ceph-mon[29191]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:49:14.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:14 smithi045 ceph-mon[33087]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:14.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:14 smithi045 ceph-mon[33087]: Cluster is now healthy 2024-09-21T09:49:14.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:14 smithi045 ceph-mon[33087]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:49:15.588 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:15 smithi032 ceph-mon[40061]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:49:15.588 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:15 smithi032 ceph-mon[40061]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-09-21T09:49:15.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:15 smithi032 ceph-mon[29191]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:49:15.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:15 smithi032 ceph-mon[29191]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-09-21T09:49:15.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph config log 1 --format=json 2024-09-21T09:49:15.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:15 smithi045 ceph-mon[33087]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:49:15.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:15 smithi045 ceph-mon[33087]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-09-21T09:49:17.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:17 smithi032 ceph-mon[29191]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-21T09:49:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:17 smithi032 ceph-mon[40061]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-21T09:49:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:17 smithi045 ceph-mon[33087]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-21T09:49:18.396 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:19.185 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:18 smithi032 systemd[1]: Starting Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:49:19.437 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:19 smithi032 ceph-mon[40061]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 299 B/s wr, 17 op/s 2024-09-21T09:49:19.437 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:19 smithi032 ceph-mon[29191]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 299 B/s wr, 17 op/s 2024-09-21T09:49:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:19 smithi045 ceph-mon[33087]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 299 B/s wr, 17 op/s 2024-09-21T09:49:20.103 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:49:20.344 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3321395080' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:49:20.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3321395080' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:49:20.344 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 podman[68641]: 2024-09-21 09:49:20.070825842 +0000 UTC m=+0.112248952 volume create dd057108f03e0276d1b591d2ef08363d85cb1e291b7410a6155fd30a581996a8 2024-09-21T09:49:20.344 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 podman[68641]: 2024-09-21 09:49:19.975580439 +0000 UTC m=+0.017003560 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-21T09:49:20.344 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 podman[68641]: 2024-09-21 09:49:20.180154138 +0000 UTC m=+0.221577290 container create 94962fa133e7e3bd2661f937d3c6ded2ac7c8c66bef854d06eb49989b23f12c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:20.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:20 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3321395080' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 podman[68641]: 2024-09-21 09:49:20.650150676 +0000 UTC m=+0.691573796 container init 94962fa133e7e3bd2661f937d3c6ded2ac7c8c66bef854d06eb49989b23f12c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 podman[68641]: 2024-09-21 09:49:20.653561062 +0000 UTC m=+0.694984176 container start 94962fa133e7e3bd2661f937d3c6ded2ac7c8c66bef854d06eb49989b23f12c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:20.666Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:20.666Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:20.668Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.32 port=9094 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:20.669Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:20.691Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:20.692Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:20.695Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:20.695Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 bash[68641]: 94962fa133e7e3bd2661f937d3c6ded2ac7c8c66bef854d06eb49989b23f12c2 2024-09-21T09:49:20.932 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:20 smithi032 systemd[1]: Started Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:21.191 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:20 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:49:20] ENGINE Bus STOPPING 2024-09-21T09:49:21.191 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:49:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:49:21.191 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:49:21] ENGINE Bus STOPPED 2024-09-21T09:49:21.191 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:49:21] ENGINE Bus STARTING 2024-09-21T09:49:21.192 INFO:teuthology.orchestra.run.smithi032.stdout:[{"version":21,"timestamp":"2024-09-21T09:49:10.057748+0000","name":"","changes":[{"name":"osd/host:smithi045/osd_memory_target","previous_value":"4480655052","new_value":"4212219596"}]}] 2024-09-21T09:49:21.192 INFO:tasks.ceph_manager:config epoch is 21 2024-09-21T09:49:21.192 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:49:21.192 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:49:21.192 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:49:21.397 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:21.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:49:21] ENGINE Serving on http://:::9283 2024-09-21T09:49:21.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:49:21] ENGINE Bus STARTED 2024-09-21T09:49:22.170 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.170 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.170 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: Deploying daemon grafana.a on smithi045 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[40061]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 255 B/s wr, 14 op/s 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.171 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T09:49:22.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:49:22.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:49:22.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: Deploying daemon grafana.a on smithi045 2024-09-21T09:49:22.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:21 smithi032 ceph-mon[29191]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 255 B/s wr, 14 op/s 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: Deploying daemon grafana.a on smithi045 2024-09-21T09:49:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:21 smithi045 ceph-mon[33087]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 255 B/s wr, 14 op/s 2024-09-21T09:49:22.806 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:22 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:22.669Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00030099s 2024-09-21T09:49:23.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:22 smithi032 ceph-mon[29191]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:49:23.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:22 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:22 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:22 smithi032 ceph-mon[40061]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:49:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:22 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:22 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:23.240 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:49:23.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:22 smithi045 ceph-mon[33087]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:49:23.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:22 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:23.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:22 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:23.919 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:49:23] "GET /metrics HTTP/1.1" 200 28497 "" "Prometheus/2.43.0" 2024-09-21T09:49:23.945 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":20,"flags":0,"active_gid":24335,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":431176591},{"type":"v1","addr":"172.21.15.32:6801","nonce":431176591}]},"active_addr":"172.21.15.32:6801/431176591","active_change":"2024-09-21T09:49:07.126081+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24346,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.32:8443/","prometheus":"http://172.21.15.32:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1200491506}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1465894634}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1858508442}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":2103196055}]}]} 2024-09-21T09:49:23.951 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:49:23.951 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:49:23.951 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:49:24.160 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:24.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:23 smithi032 ceph-mon[29191]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 232 B/s wr, 0 op/s 2024-09-21T09:49:24.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:23 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/238322128' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:49:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:23 smithi032 ceph-mon[40061]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 232 B/s wr, 0 op/s 2024-09-21T09:49:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:23 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/238322128' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:49:24.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:23 smithi045 ceph-mon[33087]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 232 B/s wr, 0 op/s 2024-09-21T09:49:24.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:23 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/238322128' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:49:25.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:25 smithi032 ceph-mon[40061]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:49:25.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:25 smithi032 ceph-mon[29191]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:49:25.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:25 smithi045 ceph-mon[33087]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:49:25.962 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:49:25.962 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":66,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:43:37.317307+0000","modified":"2024-09-21T09:49:21.931772+0000","last_up_change":"2024-09-21T09:48:34.864213+0000","last_in_change":"2024-09-21T09:48:18.335792+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:46:40.328691+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:49:08.740859+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0f6f5dda-c134-4283-a91d-cf501ef19062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6803","nonce":2765215923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6805","nonce":2765215923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6809","nonce":2765215923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6807","nonce":2765215923}]},"public_addr":"172.21.15.32:6803/2765215923","cluster_addr":"172.21.15.32:6805/2765215923","heartbeat_back_addr":"172.21.15.32:6809/2765215923","heartbeat_front_addr":"172.21.15.32:6807/2765215923","state":["exists","up"]},{"osd":1,"uuid":"bdaa43fd-b969-4d06-b6eb-97b415c4e118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6811","nonce":3808018383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6813","nonce":3808018383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6817","nonce":3808018383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6815","nonce":3808018383}]},"public_addr":"172.21.15.32:6811/3808018383","cluster_addr":"172.21.15.32:6813/3808018383","heartbeat_back_addr":"172.21.15.32:6817/3808018383","heartbeat_front_addr":"172.21.15.32:6815/3808018383","state":["exists","up"]},{"osd":2,"uuid":"1f12d9f2-90b0-44f6-8432-0b398194b3b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6819","nonce":2468805298}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6821","nonce":2468805298}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6825","nonce":2468805298}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6823","nonce":2468805298}]},"public_addr":"172.21.15.32:6819/2468805298","cluster_addr":"172.21.15.32:6821/2468805298","heartbeat_back_addr":"172.21.15.32:6825/2468805298","heartbeat_front_addr":"172.21.15.32:6823/2468805298","state":["exists","up"]},{"osd":3,"uuid":"6a568fd5-5fd9-46f8-b3af-9fa196ed617b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6827","nonce":2765373470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6829","nonce":2765373470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6833","nonce":2765373470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6831","nonce":2765373470}]},"public_addr":"172.21.15.32:6827/2765373470","cluster_addr":"172.21.15.32:6829/2765373470","heartbeat_back_addr":"172.21.15.32:6833/2765373470","heartbeat_front_addr":"172.21.15.32:6831/2765373470","state":["exists","up"]},{"osd":4,"uuid":"2cf149e0-4c2f-46eb-9ca2-a51907aea3b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6801","nonce":413300252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6803","nonce":413300252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6807","nonce":413300252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6805","nonce":413300252}]},"public_addr":"172.21.15.45:6801/413300252","cluster_addr":"172.21.15.45:6803/413300252","heartbeat_back_addr":"172.21.15.45:6807/413300252","heartbeat_front_addr":"172.21.15.45:6805/413300252","state":["exists","up"]},{"osd":5,"uuid":"a6c78855-6756-4b0b-9a9c-8d102079540b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6809","nonce":3168849729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6811","nonce":3168849729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6815","nonce":3168849729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6813","nonce":3168849729}]},"public_addr":"172.21.15.45:6809/3168849729","cluster_addr":"172.21.15.45:6811/3168849729","heartbeat_back_addr":"172.21.15.45:6815/3168849729","heartbeat_front_addr":"172.21.15.45:6813/3168849729","state":["exists","up"]},{"osd":6,"uuid":"d476169d-dffc-4f1e-b60b-4f836f53285e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6817","nonce":2158139679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6819","nonce":2158139679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6823","nonce":2158139679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6821","nonce":2158139679}]},"public_addr":"172.21.15.45:6817/2158139679","cluster_addr":"172.21.15.45:6819/2158139679","heartbeat_back_addr":"172.21.15.45:6823/2158139679","heartbeat_front_addr":"172.21.15.45:6821/2158139679","state":["exists","up"]},{"osd":7,"uuid":"021a4fd0-f770-4799-9683-b0aa7cd593bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6825","nonce":3336053350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6827","nonce":3336053350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6831","nonce":3336053350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6829","nonce":3336053350}]},"public_addr":"172.21.15.45:6825/3336053350","cluster_addr":"172.21.15.45:6827/3336053350","heartbeat_back_addr":"172.21.15.45:6831/3336053350","heartbeat_front_addr":"172.21.15.45:6829/3336053350","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:50.697928+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:13.831263+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:37.360357+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:00.010474+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:21.607129+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:45.583237+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:09.486296+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:33.456690+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6800/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/1259145858":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/230160407":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/3332221400":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/1178553739":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/458122323":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/2895073807":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6801/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/533058733":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2042031504":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2388392475":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/2785183325":"2024-09-22T09:49:07.125921+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:49:26.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:26 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/2936746499' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:49:26.589 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:26 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/2936746499' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:49:26.590 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:26 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2936746499' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:49:26.612 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:49:26.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:49:26.814 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:27.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:27 smithi032 ceph-mon[40061]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:49:27.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:27 smithi032 ceph-mon[29191]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:49:27.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:27 smithi045 ceph-mon[33087]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:49:28.432 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:49:28.432 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":66,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:43:37.317307+0000","modified":"2024-09-21T09:49:21.931772+0000","last_up_change":"2024-09-21T09:48:34.864213+0000","last_in_change":"2024-09-21T09:48:18.335792+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:46:40.328691+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:49:08.740859+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0f6f5dda-c134-4283-a91d-cf501ef19062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6803","nonce":2765215923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6805","nonce":2765215923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6809","nonce":2765215923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6807","nonce":2765215923}]},"public_addr":"172.21.15.32:6803/2765215923","cluster_addr":"172.21.15.32:6805/2765215923","heartbeat_back_addr":"172.21.15.32:6809/2765215923","heartbeat_front_addr":"172.21.15.32:6807/2765215923","state":["exists","up"]},{"osd":1,"uuid":"bdaa43fd-b969-4d06-b6eb-97b415c4e118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6811","nonce":3808018383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6813","nonce":3808018383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6817","nonce":3808018383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6815","nonce":3808018383}]},"public_addr":"172.21.15.32:6811/3808018383","cluster_addr":"172.21.15.32:6813/3808018383","heartbeat_back_addr":"172.21.15.32:6817/3808018383","heartbeat_front_addr":"172.21.15.32:6815/3808018383","state":["exists","up"]},{"osd":2,"uuid":"1f12d9f2-90b0-44f6-8432-0b398194b3b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6819","nonce":2468805298}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6821","nonce":2468805298}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6825","nonce":2468805298}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6823","nonce":2468805298}]},"public_addr":"172.21.15.32:6819/2468805298","cluster_addr":"172.21.15.32:6821/2468805298","heartbeat_back_addr":"172.21.15.32:6825/2468805298","heartbeat_front_addr":"172.21.15.32:6823/2468805298","state":["exists","up"]},{"osd":3,"uuid":"6a568fd5-5fd9-46f8-b3af-9fa196ed617b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6827","nonce":2765373470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6829","nonce":2765373470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6833","nonce":2765373470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6831","nonce":2765373470}]},"public_addr":"172.21.15.32:6827/2765373470","cluster_addr":"172.21.15.32:6829/2765373470","heartbeat_back_addr":"172.21.15.32:6833/2765373470","heartbeat_front_addr":"172.21.15.32:6831/2765373470","state":["exists","up"]},{"osd":4,"uuid":"2cf149e0-4c2f-46eb-9ca2-a51907aea3b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6801","nonce":413300252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6803","nonce":413300252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6807","nonce":413300252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6805","nonce":413300252}]},"public_addr":"172.21.15.45:6801/413300252","cluster_addr":"172.21.15.45:6803/413300252","heartbeat_back_addr":"172.21.15.45:6807/413300252","heartbeat_front_addr":"172.21.15.45:6805/413300252","state":["exists","up"]},{"osd":5,"uuid":"a6c78855-6756-4b0b-9a9c-8d102079540b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6809","nonce":3168849729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6811","nonce":3168849729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6815","nonce":3168849729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6813","nonce":3168849729}]},"public_addr":"172.21.15.45:6809/3168849729","cluster_addr":"172.21.15.45:6811/3168849729","heartbeat_back_addr":"172.21.15.45:6815/3168849729","heartbeat_front_addr":"172.21.15.45:6813/3168849729","state":["exists","up"]},{"osd":6,"uuid":"d476169d-dffc-4f1e-b60b-4f836f53285e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6817","nonce":2158139679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6819","nonce":2158139679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6823","nonce":2158139679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6821","nonce":2158139679}]},"public_addr":"172.21.15.45:6817/2158139679","cluster_addr":"172.21.15.45:6819/2158139679","heartbeat_back_addr":"172.21.15.45:6823/2158139679","heartbeat_front_addr":"172.21.15.45:6821/2158139679","state":["exists","up"]},{"osd":7,"uuid":"021a4fd0-f770-4799-9683-b0aa7cd593bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6825","nonce":3336053350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6827","nonce":3336053350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6831","nonce":3336053350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6829","nonce":3336053350}]},"public_addr":"172.21.15.45:6825/3336053350","cluster_addr":"172.21.15.45:6827/3336053350","heartbeat_back_addr":"172.21.15.45:6831/3336053350","heartbeat_front_addr":"172.21.15.45:6829/3336053350","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:50.697928+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:13.831263+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:37.360357+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:00.010474+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:21.607129+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:45.583237+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:09.486296+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:33.456690+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6800/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/1259145858":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/230160407":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/3332221400":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/1178553739":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/458122323":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/2895073807":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6801/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/533058733":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2042031504":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2388392475":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/2785183325":"2024-09-22T09:49:07.125921+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:49:28.616 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:28 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3565557165' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:49:28.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:28 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3565557165' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:49:28.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:28 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3565557165' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:49:29.002 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:49:29.003 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:49:29.003 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:49:29.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:49:29.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:49:29.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:49:29.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:49:29.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:49:29.436 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:29.437 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:29.441 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:29.441 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:29.445 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:29.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:29 smithi045 ceph-mon[33087]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:29.685 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:29.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:29 smithi032 ceph-mon[29191]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:29.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:29 smithi032 ceph-mon[40061]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:30.739 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:49:30 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:49:30.672Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003477746s 2024-09-21T09:49:30.970 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:30.985 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:30 smithi045 systemd[1]: Starting Ceph grafana.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:49:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-mon[33087]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:31.285 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:30 smithi045 podman[61475]: 2024-09-21 09:49:30.885541872 +0000 UTC m=+0.015441493 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-21T09:49:31.285 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:30 smithi045 podman[61475]: 2024-09-21 09:49:30.994669083 +0000 UTC m=+0.124568704 container create 419a19b409916fccd763e8b844c03916bb45227a1cc2fd7a0f0555c1cf3b3e19 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, maintainer=Paul Cuzner , distribution-scope=public, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.openshift.expose-services=) 2024-09-21T09:49:31.285 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 podman[61475]: 2024-09-21 09:49:31.213908269 +0000 UTC m=+0.343807904 container init 419a19b409916fccd763e8b844c03916bb45227a1cc2fd7a0f0555c1cf3b3e19 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, architecture=x86_64, distribution-scope=public, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, name=ubi8) 2024-09-21T09:49:31.285 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 podman[61475]: 2024-09-21 09:49:31.216919334 +0000 UTC m=+0.346818954 container start 419a19b409916fccd763e8b844c03916bb45227a1cc2fd7a0f0555c1cf3b3e19 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, io.buildah.version=1.32.0, vendor=Red Hat, Inc., version=8.5, distribution-scope=public, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, name=ubi8) 2024-09-21T09:49:31.377 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:31 smithi032 ceph-mon[29191]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:31.377 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:31 smithi032 ceph-mon[40061]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:31.558 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 bash[61475]: 419a19b409916fccd763e8b844c03916bb45227a1cc2fd7a0f0555c1cf3b3e19 2024-09-21T09:49:31.558 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 systemd[1]: Started Ceph grafana.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:31.558 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387360856Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-21T09:49:31.558 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387787438Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-21T09:49:31.558 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387815316Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:49:31.558 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387831774Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387845706Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387859049Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387871939Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.38788506Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387898257Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387912406Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387928394Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387941155Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387954504Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=settings t=2024-09-21T09:49:31.387968377Z level=info msg="App mode production" 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=sqlstore t=2024-09-21T09:49:31.388035567Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=sqlstore t=2024-09-21T09:49:31.388062443Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:49:31.559 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.388971813Z level=info msg="Starting DB migrations" 2024-09-21T09:49:31.560 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.389062639Z level=info msg="Executing migration" id="create migration_log table" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.389596494Z level=info msg="Executing migration" id="create user table" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.390047824Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.390501318Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.39093532Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.391329386Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.391718614Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.392384612Z level=info msg="Executing migration" id="create user table v2" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.392836018Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.393185322Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.393612825Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.393934205Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-21T09:49:31.561 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.394237635Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.394635409Z level=info msg="Executing migration" id="Update user table charset" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.394916456Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.395272421Z level=info msg="Executing migration" id="Add missing user data" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.395613582Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.396022656Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.396368634Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.396794277Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.397920429Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.398340927Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.398777067Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.399181183Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.399595077Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-21T09:49:31.562 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.400000241Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.400242665Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.400619172Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.401009022Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.401335236Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.401706734Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.402271802Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.402659002Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.403057826Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.403417482Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.40380137Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.404196801Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.404541321Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.404861364Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-21T09:49:31.563 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.40514119Z level=info msg="Executing migration" id="create star table" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.405485778Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.405935354Z level=info msg="Executing migration" id="create org table v1" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.406402003Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.406836056Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.407234725Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.40766298Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.40810501Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.408532526Z level=info msg="Executing migration" id="Update org table charset" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.4088224Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.409087944Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.40941547Z level=info msg="Executing migration" id="create dashboard table" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.409821418Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-21T09:49:31.564 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.41024335Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.410695225Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.411109419Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.411562041Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.411986321Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.412863261Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.413276661Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.413730581Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.414179108Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.414530727Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.41494778Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.415219082Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.415717316Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-21T09:49:31.565 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.416215859Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.416731634Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.417162814Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.417675926Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.418117477Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.418561673Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.418875821Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.419138112Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.419676887Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.420187886Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.420694006Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.421202741Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.421578625Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.422047525Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-21T09:49:31.566 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.422423792Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.422752839Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.423148069Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.423551977Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.42448977Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.424890933Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.425286846Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.425726783Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.426027878Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.426327035Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.426815495Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.427172181Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.427432296Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-21T09:49:31.567 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.427718859Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.428102068Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.428621258Z level=info msg="Executing migration" id="create data_source table" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.429015783Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.429415858Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.429808729Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.430150629Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.430533613Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.431520597Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.431931264Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.432289339Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.432688068Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.433028436Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.433545738Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-21T09:49:31.568 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.434039147Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.434281266Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.434573049Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.435079524Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.435337252Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.435632217Z level=info msg="Executing migration" id="Add uid column" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.436102819Z level=info msg="Executing migration" id="Update uid value" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.436360567Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.436777428Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.437163594Z level=info msg="Executing migration" id="create api_key table" 2024-09-21T09:49:31.569 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.437626454Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-21T09:49:31.571 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.438069423Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-21T09:49:31.571 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.438537876Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-21T09:49:31.571 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.43900236Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-21T09:49:31.571 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.439379984Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-21T09:49:31.571 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.439797514Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-21T09:49:31.571 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.440176522Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-21T09:49:31.571 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.441242742Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.441664328Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.442092363Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.442524696Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.442953316Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.443256325Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.443626011Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.443931013Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.444452944Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.444998558Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.445283271Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.445855693Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.446368581Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-21T09:49:31.572 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.446806471Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.447155574Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.447588087Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.448027785Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.448450837Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.448890965Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.449181328Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.449498585Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.450082655Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.450676469Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.451047819Z level=info msg="Executing migration" id="create quota table v1" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.451471217Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.451933724Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.452207731Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.452633259Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-21T09:49:31.573 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.453073778Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.453664679Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.453981707Z level=info msg="Executing migration" id="create session table" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.454438549Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.454774465Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.455051941Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.455498731Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.45591848Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.456188354Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.456492411Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.456818976Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.457099615Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.457528263Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-21T09:49:31.574 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.457846668Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.458424479Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.458759887Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.459317945Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.459986621Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.460265217Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.460710261Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.461200795Z level=info msg="Executing migration" id="create alert table v1" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.461680912Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.462158331Z level=info msg="Executing migration" id="add index alert state" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.462593208Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.463026363Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.463452998Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-21T09:49:31.575 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.463899152Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.464273529Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.465718422Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.466115658Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.466565393Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.46694964Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.46728405Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.467706118Z level=info msg="Executing migration" id="Add column is_default" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.468299766Z level=info msg="Executing migration" id="Add column frequency" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.468963726Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.469619695Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.470263428Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.47075076Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.47107694Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-21T09:49:31.576 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.471351471Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.471816753Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.472278176Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.47270052Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.473148653Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.473599082Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.474260899Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.474919158Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.475282504Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.475761888Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.476207201Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.47688298Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.477220566Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.477682148Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.478147441Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-21T09:49:31.577 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.478450189Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.478922412Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.479386011Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.479849536Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.480303305Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.480830466Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.481308569Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.481649411Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.482279123Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.482724915Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.483418633Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.483840553Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.484283864Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-21T09:49:31.578 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.484716278Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.486313311Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.486734093Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.487175622Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.487509158Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.487874313Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.488168216Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.488877923Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.489569832Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.490012956Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.490477009Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.490800805Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.49146191Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.491902932Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-21T09:49:31.579 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.492199633Z level=info msg="Executing migration" id="Move region to single row" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.492586934Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.493004813Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.493420025Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.493859766Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.494250867Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.494703339Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.495131653Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.495441107Z level=info msg="Executing migration" id="create test_data table" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.495865063Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.49626554Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.496748426Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.497410414Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-21T09:49:31.580 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.497739735Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.498040151Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.498311356Z level=info msg="Executing migration" id="create team table" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.498736069Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.499183442Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.499640026Z level=info msg="Executing migration" id="create team member table" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.500034553Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.500466749Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.500909601Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.501328353Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.50206897Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.502783309Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.503509204Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.503920177Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-21T09:49:31.581 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.504404004Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.504844631Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.505270967Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.505764379Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.506191912Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.506630382Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.50705571Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.507452361Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.507778009Z level=info msg="Executing migration" id="create tag table" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.508174521Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.508618034Z level=info msg="Executing migration" id="create login attempt table" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.509011373Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.50946211Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.509938547Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-21T09:49:31.582 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.511859231Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.512257614Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.512701379Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.513034672Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.513360945Z level=info msg="Executing migration" id="create user auth table" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.513769528Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.514214801Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.514518999Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.51528015Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.516020687Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.516773253Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.517515534Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.517960001Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.518700315Z level=info msg="Executing migration" id="create server_lock table" 2024-09-21T09:49:31.583 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.51910777Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.519563563Z level=info msg="Executing migration" id="create user auth token table" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.520028331Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.520454792Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.520913053Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.521344536Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.522147206Z level=info msg="Executing migration" id="create cache_data table" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.522576898Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.523017726Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.523477697Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.523918032Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.524190597Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.524506842Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-21T09:49:31.584 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.524950438Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.525415198Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.525852574Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.526124375Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.52655251Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.526976876Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.527405192Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.527858463Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.528662789Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.529131054Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.529425907Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.529857742Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.530266942Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.530710335Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.531024569Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-21T09:49:31.585 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.531441778Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.531902954Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.532309946Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.532776921Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.533574159Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.53401724Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.534413674Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.537878658Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.541335517Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.541820931Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.542227293Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.54307351Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.543557922Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.544027996Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-21T09:49:31.586 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.544500847Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.544959595Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.54522961Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.546074386Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.546907392Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.547758723Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.548186975Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.548638056Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.549449984Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.550261658Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.550713987Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.551543379Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.552394751Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.552734336Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-21T09:49:31.587 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.553196634Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.553677757Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.554142182Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.554424337Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.555298581Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.556140578Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.557049867Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.557930053Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.558804582Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.559115202Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.559540651Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.560534615Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.560854257Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.561798953Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.56225343Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-21T09:49:31.588 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.563143448Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-21T09:49:31.589 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.563568401Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-21T09:49:31.589 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.564019086Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-21T09:49:31.589 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.564924082Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-21T09:49:31.589 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.565344162Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-21T09:49:31.589 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.565801179Z level=info msg="Executing migration" id="create alert_image table" 2024-09-21T09:49:31.589 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.56623589Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-21T09:49:31.589 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.566750152Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-21T09:49:31.589 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.567092318Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.567574659Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.568016905Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.568106822Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.5683497Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.568749061Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.569155649Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.569289164Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.569427196Z level=info msg="alerts found to migrate" alerts=0 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.569847057Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.570271119Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.570733725Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-21T09:49:31.590 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.571120526Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-21T09:49:32.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.5716061Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-21T09:49:32.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.572118511Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-21T09:49:32.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.57236875Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-21T09:49:32.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.572767852Z level=info msg="Executing migration" id="create data_keys table" 2024-09-21T09:49:32.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.573196328Z level=info msg="Executing migration" id="create secrets table" 2024-09-21T09:49:32.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.573674671Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.578122634Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.579054989Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.579404366Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.583747972Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.588083727Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.588503803Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.588962438Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.589291405Z level=info msg="Executing migration" id="create permission table" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.589709398Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.590148417Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-21T09:49:32.036 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.590619313Z level=info msg="Executing migration" id="create role table" 2024-09-21T09:49:32.037 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.591014925Z level=info msg="Executing migration" id="add column display_name" 2024-09-21T09:49:32.037 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.591937123Z level=info msg="Executing migration" id="add column group_name" 2024-09-21T09:49:32.037 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.59285007Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-21T09:49:32.037 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.593312954Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-21T09:49:32.037 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.593779575Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.594237859Z level=info msg="Executing migration" id="create team role table" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.594640296Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.595081201Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.595565972Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.596010141Z level=info msg="Executing migration" id="create user role table" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.596437068Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.596879669Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.597318328Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.597768217Z level=info msg="Executing migration" id="create builtin role table" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.5981785Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-21T09:49:32.038 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.598634424Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.599067386Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.600033542Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.600544371Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.600995534Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.601429895Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.601873028Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.602261292Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.602755222Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.603745794Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.604154811Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.604613976Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-21T09:49:32.039 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.604910237Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.605299088Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.605797512Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.606171343Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.606452191Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.606823303Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.607202412Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.607678605Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.608641505Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.608934536Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.609345745Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.609811628Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-21T09:49:32.040 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.610259272Z level=info msg="Executing migration" id="add correlation config column" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.611231038Z level=info msg="Executing migration" id="create entity_events table" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.611669251Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.612089711Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.612187406Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.612497051Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.612618987Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.612912095Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.613314049Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.613771084Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.614244978Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.614729908Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:49:32.041 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.615183654Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:49:32.042 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.615639834Z level=info msg="Executing migration" id="Drop public config table" 2024-09-21T09:49:32.042 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.61611952Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-21T09:49:32.042 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.616569246Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:49:32.042 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.617038707Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:49:32.042 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.617523258Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-21T09:49:32.042 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.617992463Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.621077477Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.622113419Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.623240212Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.623589753Z level=info msg="Executing migration" id="add share column" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.624624308Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.625079765Z level=info msg="Executing migration" id="create file table" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.625701421Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.626170903Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.626684733Z level=info msg="Executing migration" id="create file_meta table" 2024-09-21T09:49:32.043 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.627104318Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.627562329Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.627859647Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.628212214Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.628551013Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.629000397Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.630038001Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.630358902Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.630824062Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.631199685Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.63153693Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.631909255Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-21T09:49:32.044 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.632937845Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.633961186Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.63441814Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.644434525Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.644899839Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.645347383Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.648932341Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.649287371Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.649642927Z level=info msg="Executing migration" id="create folder table" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.650063818Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.650536021Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-21T09:49:32.045 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.650990111Z level=info msg="Executing migration" id="Update folder title length" 2024-09-21T09:49:32.046 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.651257703Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-21T09:49:32.046 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=migrator t=2024-09-21T09:49:31.651701899Z level=info msg="migrations completed" performed=485 skipped=0 duration=262.658579ms 2024-09-21T09:49:32.046 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=sqlstore t=2024-09-21T09:49:31.652127487Z level=info msg="Created default organization" 2024-09-21T09:49:32.046 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=plugin.loader t=2024-09-21T09:49:31.712717444Z level=info msg="Plugin registered" pluginID=input 2024-09-21T09:49:32.046 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=plugin.loader t=2024-09-21T09:49:31.74206613Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-21T09:49:32.046 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=plugin.loader t=2024-09-21T09:49:31.742084584Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-21T09:49:32.046 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=secrets t=2024-09-21T09:49:31.742117693Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:49:32.046 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=query_data t=2024-09-21T09:49:31.743811078Z level=info msg="Query Service initialization" 2024-09-21T09:49:32.046 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=live.push_http t=2024-09-21T09:49:31.74611397Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:49:32.152 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:32.518 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.519 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.519 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.519 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.519 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[40061]: Deploying daemon node-exporter.a on smithi032 2024-09-21T09:49:32.519 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.519 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.519 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.519 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.519 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.520 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[29191]: Deploying daemon node-exporter.a on smithi032 2024-09-21T09:49:32.520 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:32 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.629 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.629 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.629 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.629 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.629 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-mon[33087]: Deploying daemon node-exporter.a on smithi032 2024-09-21T09:49:32.630 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:32.630 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=infra.usagestats.collector t=2024-09-21T09:49:32.512687718Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:49:32.630 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=provisioning.datasources t=2024-09-21T09:49:32.568540529Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:49:32.630 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=provisioning.datasources t=2024-09-21T09:49:32.600762584Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-21T09:49:32.630 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=provisioning.alerting t=2024-09-21T09:49:32.622889218Z level=info msg="starting to provision alerting" 2024-09-21T09:49:32.630 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=provisioning.alerting t=2024-09-21T09:49:32.622909911Z level=info msg="finished to provision alerting" 2024-09-21T09:49:32.630 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=grafanaStorageLogger t=2024-09-21T09:49:32.623008862Z level=info msg="storage starting" 2024-09-21T09:49:32.630 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=ngalert.state.manager t=2024-09-21T09:49:32.623018661Z level=info msg="Warming state cache for startup" 2024-09-21T09:49:33.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=http.server t=2024-09-21T09:49:32.628136118Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:49:33.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=ngalert.state.manager t=2024-09-21T09:49:32.633299079Z level=info msg="State cache has been initialized" states=0 duration=10.277628ms 2024-09-21T09:49:33.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:49:32.633358162Z level=info msg="starting MultiOrg Alertmanager" 2024-09-21T09:49:33.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=ticker t=2024-09-21T09:49:32.633393058Z level=info msg=starting first_tick=2024-09-21T09:49:40Z 2024-09-21T09:49:33.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=sqlstore.transactions t=2024-09-21T09:49:32.633401968Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:49:33.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=sqlstore.transactions t=2024-09-21T09:49:32.679170476Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:49:33.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:49:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[61490]: logger=sqlstore.transactions t=2024-09-21T09:49:32.68962614Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-21T09:49:33.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:33 smithi032 ceph-mon[29191]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:33.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:33 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:49:33] "GET /metrics HTTP/1.1" 200 28497 "" "Prometheus/2.43.0" 2024-09-21T09:49:33.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:33 smithi032 ceph-mon[40061]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:33.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:33 smithi045 ceph-mon[33087]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:35.409 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:35 smithi032 ceph-mon[40061]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:35.410 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:35 smithi032 ceph-mon[29191]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:35.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:35 smithi045 ceph-mon[33087]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:36.430 INFO:teuthology.orchestra.run.smithi032.stdout:128849018908 2024-09-21T09:49:36.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:49:37.524 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:37 smithi032 ceph-mon[29191]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:37.525 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:37 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:37.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:37 smithi032 ceph-mon[40061]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:37.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:37 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:37.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:37 smithi045 ceph-mon[33087]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:37.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:37 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:39.271 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:39 smithi032 ceph-mon[40061]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:39.271 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:39 smithi032 ceph-mon[29191]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:39.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:39 smithi045 ceph-mon[33087]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:40.193 INFO:teuthology.orchestra.run.smithi032.stdout:55834574892 2024-09-21T09:49:40.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:49:40.856 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:40 smithi032 systemd[1]: Starting Ceph node-exporter.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:49:41.470 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:41 smithi032 ceph-mon[40061]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:41.470 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:41 smithi032 ceph-mon[29191]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:41.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:41 smithi045 ceph-mon[33087]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:43 smithi045 ceph-mon[33087]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:43.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:43 smithi032 ceph-mon[29191]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:43.627 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:43 smithi032 ceph-mon[40061]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:43.850 INFO:teuthology.orchestra.run.smithi032.stdout:107374182434 2024-09-21T09:49:43.850 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:49:43.877 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:43 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:49:43] "GET /metrics HTTP/1.1" 200 28433 "" "Prometheus/2.43.0" 2024-09-21T09:49:44.441 INFO:teuthology.orchestra.run.smithi032.stdout:163208757273 2024-09-21T09:49:44.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:49:44.791 INFO:teuthology.orchestra.run.smithi032.stdout:77309411366 2024-09-21T09:49:44.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:49:45.364 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:45 smithi032 ceph-mon[40061]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:45.365 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:45 smithi032 ceph-mon[29191]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:45.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:45 smithi045 ceph-mon[33087]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:45.898 INFO:teuthology.orchestra.run.smithi032.stdout:231928233998 2024-09-21T09:49:45.899 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:49:45.917 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:46.206 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:46 smithi032 bash[70592]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T09:49:46.576 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:46 smithi032 bash[70592]: Getting image source signatures 2024-09-21T09:49:46.576 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:46 smithi032 bash[70592]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T09:49:46.576 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:46 smithi032 bash[70592]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T09:49:46.576 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:46 smithi032 bash[70592]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T09:49:46.612 INFO:teuthology.orchestra.run.smithi032.stdout:197568495637 2024-09-21T09:49:46.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:49:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:47 smithi045 ceph-mon[33087]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:47.682 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:47 smithi032 ceph-mon[40061]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:47.683 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:47 smithi032 ceph-mon[29191]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:48.206 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:47 smithi032 bash[70592]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T09:49:48.557 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:48 smithi032 bash[70592]: Writing manifest to image destination 2024-09-21T09:49:48.834 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:48 smithi032 podman[70592]: 2024-09-21 09:49:48.549370374 +0000 UTC m=+5.189383552 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:49:48.856 INFO:teuthology.orchestra.run.smithi032.stdout:34359738417 2024-09-21T09:49:48.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:49:48.868 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:49.523 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-mon[29191]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:49.523 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-mon[40061]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:49.524 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 podman[70592]: 2024-09-21 09:49:49.361045775 +0000 UTC m=+6.001058940 container create e4bf1511c58419374805070c66e12ee96d49d4f1bd25c314dcbde3bcab9bd26f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:49.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:49 smithi045 ceph-mon[33087]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:49.957 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:50.085 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 podman[70592]: 2024-09-21 09:49:49.882673847 +0000 UTC m=+6.522687023 container init e4bf1511c58419374805070c66e12ee96d49d4f1bd25c314dcbde3bcab9bd26f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:50.085 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 podman[70592]: 2024-09-21 09:49:49.886192228 +0000 UTC m=+6.526205393 container start e4bf1511c58419374805070c66e12ee96d49d4f1bd25c314dcbde3bcab9bd26f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:50.085 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.890Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:49:50.085 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.890Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.891Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.891Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.891Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.891Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.891Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.891Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.891Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.891Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.891Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:49:50.086 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:49:50.087 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:49:50.087 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:49:50.087 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:49:50.087 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:49:50.087 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:49:50.087 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:49:50.087 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:49:50.087 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:49:50.088 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:49:50.089 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:49:50.090 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:49:50.090 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:49:50.090 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[71164]: ts=2024-09-21T09:49:49.892Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:49:50.090 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 bash[70592]: e4bf1511c58419374805070c66e12ee96d49d4f1bd25c314dcbde3bcab9bd26f 2024-09-21T09:49:50.090 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:49:49 smithi032 systemd[1]: Started Ceph node-exporter.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:50.780 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:50 smithi045 systemd[1]: Starting Ceph node-exporter.b for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:49:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:51 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:51 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:51 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:51 smithi045 ceph-mon[33087]: Deploying daemon node-exporter.b on smithi045 2024-09-21T09:49:51.036 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:50 smithi045 bash[61756]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T09:49:51.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:51 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:51.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:51 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:51 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:51 smithi032 ceph-mon[29191]: Deploying daemon node-exporter.b on smithi045 2024-09-21T09:49:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:51 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:51 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:51 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:51 smithi032 ceph-mon[40061]: Deploying daemon node-exporter.b on smithi045 2024-09-21T09:49:51.785 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:51 smithi045 bash[61756]: Getting image source signatures 2024-09-21T09:49:51.785 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:51 smithi045 bash[61756]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T09:49:51.785 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:51 smithi045 bash[61756]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T09:49:51.785 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:51 smithi045 bash[61756]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T09:49:52.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:52 smithi032 ceph-mon[29191]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:52.211 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:52 smithi032 ceph-mon[40061]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:52.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:52 smithi045 ceph-mon[33087]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:52.793 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:52.795 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:52.796 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:52.974 INFO:teuthology.orchestra.run.smithi032.stdout:128849018911 2024-09-21T09:49:52.991 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:52 smithi045 bash[61756]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T09:49:52.991 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:52 smithi045 bash[61756]: Writing manifest to image destination 2024-09-21T09:49:53.165 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:53 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:53.165 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:53 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3760360612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:49:53.165 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:53 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:53.165 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:53 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3760360612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:49:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3760360612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:49:53.551 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 podman[61756]: 2024-09-21 09:49:53.262837775 +0000 UTC m=+2.386552009 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:49:53.551 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 podman[61756]: 2024-09-21 09:49:53.344708251 +0000 UTC m=+2.468422472 container create 49083320e62a4b2a69642b805b2f7ee97c26752e10c59890dd131991c3378043 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:49:53.552 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 podman[61756]: 2024-09-21 09:49:53.547279893 +0000 UTC m=+2.670994116 container init 49083320e62a4b2a69642b805b2f7ee97c26752e10c59890dd131991c3378043 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:49:53.552 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 podman[61756]: 2024-09-21 09:49:53.550335476 +0000 UTC m=+2.674049697 container start 49083320e62a4b2a69642b805b2f7ee97c26752e10c59890dd131991c3378043 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:49:53.889 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:49:53 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:49:53] "GET /metrics HTTP/1.1" 200 28498 "" "Prometheus/2.43.0" 2024-09-21T09:49:53.941 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.554Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:49:53.941 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.554Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:49:53.941 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.555Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:49:53.941 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.555Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:49:53.941 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.555Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:49:53.941 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:49:53.941 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:49:53.942 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:49:53.943 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:49:53.943 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:49:53.943 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:49:53.943 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:49:53.943 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:49:53.943 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:49:53.943 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:49:53.943 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:49:53.944 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.556Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.557Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:49:53.945 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[61834]: ts=2024-09-21T09:49:53.557Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:49:53.946 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 bash[61756]: 49083320e62a4b2a69642b805b2f7ee97c26752e10c59890dd131991c3378043 2024-09-21T09:49:53.946 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:49:53 smithi045 systemd[1]: Started Ceph node-exporter.b for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:54.198 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[40061]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:54.198 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.198 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.198 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.198 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.198 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.198 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:54.198 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[29191]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:54.199 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.199 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.199 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.199 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.199 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.199 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:54 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:54.204 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:54.226 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018911 for osd.4 2024-09-21T09:49:54.226 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:54.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:54 smithi045 ceph-mon[33087]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:54 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:54 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:54 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:54 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:54 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:54 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:55.523 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:55 smithi045 ceph-mon[33087]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:55.538 INFO:teuthology.orchestra.run.smithi032.stdout:107374182437 2024-09-21T09:49:55.539 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:55 smithi032 ceph-mon[40061]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:55.539 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:55 smithi032 ceph-mon[29191]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:56 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/768722212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:49:56.349 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:56 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/768722212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:49:56.349 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:56 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/768722212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:49:56.377 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182434 got 107374182437 for osd.3 2024-09-21T09:49:56.377 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:57.242 INFO:teuthology.orchestra.run.smithi032.stdout:231928234001 2024-09-21T09:49:57.430 INFO:teuthology.orchestra.run.smithi032.stdout:163208757276 2024-09-21T09:49:57.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:57 smithi045 ceph-mon[33087]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:57.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:57 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:57.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:57 smithi032 ceph-mon[29191]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:57.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:57 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:57.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:57 smithi032 ceph-mon[40061]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:57.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:57 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:57.881 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233998 got 231928234001 for osd.7 2024-09-21T09:49:57.881 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:58.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:58 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3925598791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:49:58.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:58 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:58.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:58 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:58.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:58 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3942902510' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:49:58.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:58 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3925598791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:49:58.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:58 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:58.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:58 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:58.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:58 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3942902510' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:49:58.505 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757273 got 163208757276 for osd.5 2024-09-21T09:49:58.505 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:58.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:58 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3925598791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:49:58.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:58 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:58.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:58 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:49:58.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:58 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3942902510' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:49:59.482 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:49:59 smithi032 ceph-mon[40061]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:59.482 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:49:59 smithi032 ceph-mon[29191]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:49:59.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:49:59 smithi045 ceph-mon[33087]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:00.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:00 smithi032 ceph-mon[29191]: overall HEALTH_OK 2024-09-21T09:50:00.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:00 smithi032 ceph-mon[40061]: overall HEALTH_OK 2024-09-21T09:50:00.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:00 smithi045 ceph-mon[33087]: overall HEALTH_OK 2024-09-21T09:50:01.448 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:01 smithi032 ceph-mon[29191]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:01.448 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:01 smithi032 ceph-mon[40061]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:01.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:01 smithi045 ceph-mon[33087]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:02.520 INFO:teuthology.orchestra.run.smithi032.stdout:34359738420 2024-09-21T09:50:02.695 INFO:teuthology.orchestra.run.smithi032.stdout:197568495640 2024-09-21T09:50:02.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:02 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/247403851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:50:02.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:02 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/247403851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:50:03.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:02 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/247403851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:50:03.312 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495637 got 197568495640 for osd.6 2024-09-21T09:50:03.312 DEBUG:teuthology.parallel:result is None 2024-09-21T09:50:03.659 INFO:teuthology.orchestra.run.smithi032.stdout:55834574897 2024-09-21T09:50:03.849 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:03 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/481711854' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:50:03.849 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:03 smithi032 ceph-mon[40061]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:03.849 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:03 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/481711854' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:50:03.849 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:03 smithi032 ceph-mon[29191]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:03.850 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:50:03] "GET /metrics HTTP/1.1" 200 28498 "" "Prometheus/2.43.0" 2024-09-21T09:50:04.022 INFO:teuthology.orchestra.run.smithi032.stdout:77309411370 2024-09-21T09:50:04.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:03 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/481711854' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:50:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:03 smithi045 ceph-mon[33087]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:04.458 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574897 for osd.1 2024-09-21T09:50:04.458 DEBUG:teuthology.parallel:result is None 2024-09-21T09:50:04.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:04 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1775879821' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:50:04.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:04 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3633180417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:50:04.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:04 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1775879821' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:50:04.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:04 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3633180417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:50:04.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:04 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1775879821' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:50:04.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:04 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3633180417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:50:05.218 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411370 for osd.2 2024-09-21T09:50:05.219 DEBUG:teuthology.parallel:result is None 2024-09-21T09:50:05.794 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:05 smithi032 ceph-mon[40061]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:05.795 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:05 smithi032 ceph-mon[29191]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:05.820 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738417 got 34359738420 for osd.0 2024-09-21T09:50:05.820 DEBUG:teuthology.parallel:result is None 2024-09-21T09:50:05.820 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:50:05.820 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:50:06.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:05 smithi045 ceph-mon[33087]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:06.607 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:07.363 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:07 smithi032 ceph-mon[40061]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:07.364 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:07.364 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:07 smithi032 ceph-mon[29191]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:07.364 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:07 smithi045 ceph-mon[33087]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:08.358 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:50:08.358 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T09:50:09.341 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:09 smithi032 ceph-mon[29191]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:50:09.342 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:09 smithi032 ceph-mon[29191]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:09.342 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:09 smithi032 ceph-mon[40061]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:50:09.342 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:09 smithi032 ceph-mon[40061]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:09 smithi045 ceph-mon[33087]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:50:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:09 smithi045 ceph-mon[33087]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:09.695 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-09-21T09:50:07.149283+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":235440,"kb_used_data":6992,"kb_used_omap":12,"kb_used_meta":228339,"kb_avail":749725776,"statfs":{"total":767960285184,"available":767719194624,"internally_reserved":0,"allocated":7159808,"data_stored":4035301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":233819704},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002498"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.936857+0000","last_change":"2024-09-21T09:49:14.322989+0000","last_active":"2024-09-21T09:49:21.936857+0000","last_peered":"2024-09-21T09:49:21.936857+0000","last_clean":"2024-09-21T09:49:21.936857+0000","last_became_active":"2024-09-21T09:49:10.176476+0000","last_became_peered":"2024-09-21T09:49:10.176476+0000","last_unstale":"2024-09-21T09:49:21.936857+0000","last_undegraded":"2024-09-21T09:49:21.936857+0000","last_fullsized":"2024-09-21T09:49:21.936857+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:03:19.334046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.934350+0000","last_change":"2024-09-21T09:49:14.322228+0000","last_active":"2024-09-21T09:49:21.934350+0000","last_peered":"2024-09-21T09:49:21.934350+0000","last_clean":"2024-09-21T09:49:21.934350+0000","last_became_active":"2024-09-21T09:49:10.249779+0000","last_became_peered":"2024-09-21T09:49:10.249779+0000","last_unstale":"2024-09-21T09:49:21.934350+0000","last_undegraded":"2024-09-21T09:49:21.934350+0000","last_fullsized":"2024-09-21T09:49:21.934350+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:44.504161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033889199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956221+0000","last_change":"2024-09-21T09:49:14.324021+0000","last_active":"2024-09-21T09:49:21.956221+0000","last_peered":"2024-09-21T09:49:21.956221+0000","last_clean":"2024-09-21T09:49:21.956221+0000","last_became_active":"2024-09-21T09:49:10.249765+0000","last_became_peered":"2024-09-21T09:49:10.249765+0000","last_unstale":"2024-09-21T09:49:21.956221+0000","last_undegraded":"2024-09-21T09:49:21.956221+0000","last_fullsized":"2024-09-21T09:49:21.956221+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:13.890155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031296500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.934314+0000","last_change":"2024-09-21T09:49:14.322235+0000","last_active":"2024-09-21T09:49:21.934314+0000","last_peered":"2024-09-21T09:49:21.934314+0000","last_clean":"2024-09-21T09:49:21.934314+0000","last_became_active":"2024-09-21T09:49:10.176921+0000","last_became_peered":"2024-09-21T09:49:10.176921+0000","last_unstale":"2024-09-21T09:49:21.934314+0000","last_undegraded":"2024-09-21T09:49:21.934314+0000","last_fullsized":"2024-09-21T09:49:21.934314+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:51.116299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034704499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956128+0000","last_change":"2024-09-21T09:49:14.321622+0000","last_active":"2024-09-21T09:49:21.956128+0000","last_peered":"2024-09-21T09:49:21.956128+0000","last_clean":"2024-09-21T09:49:21.956128+0000","last_became_active":"2024-09-21T09:49:10.175207+0000","last_became_peered":"2024-09-21T09:49:10.175207+0000","last_unstale":"2024-09-21T09:49:21.956128+0000","last_undegraded":"2024-09-21T09:49:21.956128+0000","last_fullsized":"2024-09-21T09:49:21.956128+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:54.922603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091875599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956079+0000","last_change":"2024-09-21T09:49:14.317504+0000","last_active":"2024-09-21T09:49:21.956079+0000","last_peered":"2024-09-21T09:49:21.956079+0000","last_clean":"2024-09-21T09:49:21.956079+0000","last_became_active":"2024-09-21T09:49:10.249697+0000","last_became_peered":"2024-09-21T09:49:10.249697+0000","last_unstale":"2024-09-21T09:49:21.956079+0000","last_undegraded":"2024-09-21T09:49:21.956079+0000","last_fullsized":"2024-09-21T09:49:21.956079+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:22.915376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039017899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956181+0000","last_change":"2024-09-21T09:49:14.324025+0000","last_active":"2024-09-21T09:49:21.956181+0000","last_peered":"2024-09-21T09:49:21.956181+0000","last_clean":"2024-09-21T09:49:21.956181+0000","last_became_active":"2024-09-21T09:49:10.177038+0000","last_became_peered":"2024-09-21T09:49:10.177038+0000","last_unstale":"2024-09-21T09:49:21.956181+0000","last_undegraded":"2024-09-21T09:49:21.956181+0000","last_fullsized":"2024-09-21T09:49:21.956181+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:22.188993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000343004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956150+0000","last_change":"2024-09-21T09:49:14.321731+0000","last_active":"2024-09-21T09:49:21.956150+0000","last_peered":"2024-09-21T09:49:21.956150+0000","last_clean":"2024-09-21T09:49:21.956150+0000","last_became_active":"2024-09-21T09:49:10.174839+0000","last_became_peered":"2024-09-21T09:49:10.174839+0000","last_unstale":"2024-09-21T09:49:21.956150+0000","last_undegraded":"2024-09-21T09:49:21.956150+0000","last_fullsized":"2024-09-21T09:49:21.956150+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:03.713393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010253300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":263,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:49:14.323700+0000","last_change":"2024-09-21T09:48:37.934335+0000","last_active":"2024-09-21T09:49:14.323700+0000","last_peered":"2024-09-21T09:49:14.323700+0000","last_clean":"2024-09-21T09:49:14.323700+0000","last_became_active":"2024-09-21T09:48:37.933834+0000","last_became_peered":"2024-09-21T09:48:37.933834+0000","last_unstale":"2024-09-21T09:49:14.323700+0000","last_undegraded":"2024-09-21T09:49:14.323700+0000","last_fullsized":"2024-09-21T09:49:14.323700+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:09.157244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28252,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716900,"statfs":{"total":95995035648,"available":95966105600,"internally_reserved":0,"allocated":1261568,"data_stored":873130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.434},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.55500000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.62}]},{"osd":4,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.68899999999999995}]},{"osd":6,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.58999999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495641,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":873480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.50800000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.432},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.59499999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.48199999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.45300000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.37},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.46800000000000003}]},{"osd":7,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.41899999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757278,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":283112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.55700000000000005,"15min":0.55500000000000005},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38900000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.55400000000000005}]},{"osd":1,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.78000000000000003,"15min":0.81000000000000005},"min":{"1min":0.40400000000000003,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.98199999999999998,"5min":7.2190000000000003,"15min":7.2190000000000003},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56200000000000006,"15min":0.56000000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75700000000000001,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.622}]},{"osd":2,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.76200000000000001,"15min":0.79600000000000004},"min":{"1min":0.39400000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80500000000000005,"5min":7.1619999999999999,"15min":7.1619999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.38400000000000001,"5min":0.376,"15min":0.376},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.57099999999999995}]},{"osd":3,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.36699999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.72099999999999997,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.81200000000000006,"15min":0.84399999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2130000000000001,"5min":7.181,"15min":7.181},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.441,"15min":0.435},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.46700000000000003,"15min":0.46100000000000002},"min":{"1min":0.316,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.754}]},{"osd":7,"last update":"Sat Sep 21 09:49:37 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.68500000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018914,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31772,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713380,"statfs":{"total":95995035648,"available":95962501120,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.438,"5min":0.313,"15min":0.313},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.76100000000000001,"15min":0.79700000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78700000000000003,"5min":5.4029999999999996,"15min":5.4029999999999996},"last":0.432}]},{"osd":1,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55800000000000005,"15min":0.56200000000000006},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.80300000000000005,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.52500000000000002},"min":{"1min":0.41199999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.51800000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52400000000000002,"15min":0.52500000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.68200000000000005,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.58799999999999997,"15min":0.58599999999999997},"min":{"1min":0.441,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.86099999999999999,"15min":0.82799999999999996},"min":{"1min":0.45800000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.0060000000000002,"5min":8.0060000000000002,"15min":8.0060000000000002},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.77800000000000002,"15min":0.80600000000000005},"min":{"1min":0.46899999999999997,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.86899999999999999,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:49:48 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.40000000000000002,"15min":0.38400000000000001},"min":{"1min":0.309,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.41499999999999998,"15min":0.39500000000000002},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.56100000000000005}]},{"osd":6,"last update":"Sat Sep 21 09:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.628}]},{"osd":7,"last update":"Sat Sep 21 09:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.503},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.66300000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182439,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.39900000000000002,"15min":0.39700000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.442,"15min":0.44400000000000001},"min":{"1min":0.28000000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45900000000000002,"15min":0.46100000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.378},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.41699999999999998,"15min":0.41499999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.29799999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.46400000000000002,"15min":0.46500000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.45700000000000002,"15min":0.46300000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.58499999999999996,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.314}]},{"osd":4,"last update":"Sat Sep 21 09:49:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.59899999999999998,"15min":0.60399999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77900000000000003,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.433},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83699999999999997,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.61299999999999999,"15min":0.61899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.82899999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76200000000000001,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:49:14 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Sep 21 09:49:38 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.496}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31772,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713380,"statfs":{"total":95995035648,"available":95962501120,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:40 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.35999999999999999,"15min":0.35399999999999998},"min":{"1min":0.23899999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.373,"15min":0.36499999999999999},"min":{"1min":0.217,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.31900000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:49:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.23599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38700000000000001,"15min":0.38100000000000001},"min":{"1min":0.30099999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.52800000000000002,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.30299999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:49:05 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.48299999999999998,"15min":0.48499999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.28699999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:49:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42599999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76100000000000001,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.59499999999999997,"15min":0.59699999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83799999999999997,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.59399999999999997,"15min":0.59799999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70599999999999996,"5min":0.878,"15min":0.878},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58499999999999996,"15min":0.58799999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.77200000000000002,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.39300000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:49:14 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:49:37 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.44},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.433}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":282839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.315},"min":{"1min":0.20799999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.34200000000000003,"15min":0.32600000000000001},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.33100000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.42399999999999999,"15min":0.40999999999999998},"min":{"1min":0.29499999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.371},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.39300000000000002,"15min":0.36799999999999999},"min":{"1min":0.27600000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.627,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.29599999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:49:05 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.39800000000000002,"15min":0.39100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.432,"15min":0.42699999999999999},"min":{"1min":0.33300000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.40000000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:49:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54400000000000004,"15min":0.54500000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74199999999999999,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.70599999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.57699999999999996,"15min":0.58299999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.75800000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:49:14 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.57799999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32352,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712800,"statfs":{"total":95995035648,"available":95961907200,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:49:24 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.34499999999999997,"15min":0.33500000000000002},"min":{"1min":0.23400000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.34799999999999998,"15min":0.33400000000000002},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.246}]},{"osd":2,"last update":"Sat Sep 21 09:49:44 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.52200000000000002,"5min":0.622,"15min":0.622},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.435,"5min":0.441,"15min":0.436},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.58999999999999997,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.375}]},{"osd":3,"last update":"Sat Sep 21 09:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.439,"15min":0.441},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.46600000000000003,"15min":0.47499999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.625,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.31}]},{"osd":4,"last update":"Sat Sep 21 09:49:28 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.60099999999999998,"15min":0.60699999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.63900000000000001,"15min":0.623},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.4210000000000003,"5min":4.4210000000000003,"15min":4.4210000000000003},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:49:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59699999999999998,"15min":0.60099999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":4.8019999999999996},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.63,"15min":0.64100000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.87,"5min":1.254,"15min":1.254},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.58699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:50:09.697 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:50:10.327 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:11.627 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:11 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:11.627 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:11 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:11.627 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:11 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:11.627 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:11 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:11.627 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:11 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:11.627 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:11 smithi045 ceph-mon[33087]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:50:11.627 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:11 smithi045 ceph-mon[33087]: Reconfiguring daemon alertmanager.a on smithi032 2024-09-21T09:50:11.627 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:11 smithi045 ceph-mon[33087]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[40061]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[40061]: Reconfiguring daemon alertmanager.a on smithi032 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[40061]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:11.662 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:11.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:11.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:11.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:11.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[29191]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:50:11.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[29191]: Reconfiguring daemon alertmanager.a on smithi032 2024-09-21T09:50:11.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:11 smithi032 ceph-mon[29191]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:13.524 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:13 smithi032 ceph-mon[40061]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:13.524 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:13 smithi032 ceph-mon[29191]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:13.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:13 smithi045 ceph-mon[33087]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:13.886 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:50:13] "GET /metrics HTTP/1.1" 200 28498 "" "Prometheus/2.43.0" 2024-09-21T09:50:14.454 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:14 smithi032 systemd[1]: Stopping Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:50:14.663 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:50:14.664 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T09:50:14.957 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:14 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[68669]: ts=2024-09-21T09:50:14.757Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:50:15.221 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:14 smithi032 podman[72996]: 2024-09-21 09:50:14.956424805 +0000 UTC m=+0.336469520 container died 94962fa133e7e3bd2661f937d3c6ded2ac7c8c66bef854d06eb49989b23f12c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:15.477 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:15 smithi032 podman[72996]: 2024-09-21 09:50:15.341443666 +0000 UTC m=+0.721488381 container cleanup 94962fa133e7e3bd2661f937d3c6ded2ac7c8c66bef854d06eb49989b23f12c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:15.477 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:15 smithi032 bash[72996]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:50:15.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:15 smithi032 ceph-mon[29191]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:50:15.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:15 smithi032 ceph-mon[29191]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:15.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:15 smithi032 ceph-mon[40061]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:50:15.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:15 smithi032 ceph-mon[40061]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:15.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:15 smithi045 ceph-mon[33087]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:50:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:15 smithi045 ceph-mon[33087]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:15.775 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-09-21T09:50:13.150270+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":235440,"kb_used_data":6992,"kb_used_omap":12,"kb_used_meta":228339,"kb_avail":749725776,"statfs":{"total":767960285184,"available":767719194624,"internally_reserved":0,"allocated":7159808,"data_stored":4035301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":233819704},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002376"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.936857+0000","last_change":"2024-09-21T09:49:14.322989+0000","last_active":"2024-09-21T09:49:21.936857+0000","last_peered":"2024-09-21T09:49:21.936857+0000","last_clean":"2024-09-21T09:49:21.936857+0000","last_became_active":"2024-09-21T09:49:10.176476+0000","last_became_peered":"2024-09-21T09:49:10.176476+0000","last_unstale":"2024-09-21T09:49:21.936857+0000","last_undegraded":"2024-09-21T09:49:21.936857+0000","last_fullsized":"2024-09-21T09:49:21.936857+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:03:19.334046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.934350+0000","last_change":"2024-09-21T09:49:14.322228+0000","last_active":"2024-09-21T09:49:21.934350+0000","last_peered":"2024-09-21T09:49:21.934350+0000","last_clean":"2024-09-21T09:49:21.934350+0000","last_became_active":"2024-09-21T09:49:10.249779+0000","last_became_peered":"2024-09-21T09:49:10.249779+0000","last_unstale":"2024-09-21T09:49:21.934350+0000","last_undegraded":"2024-09-21T09:49:21.934350+0000","last_fullsized":"2024-09-21T09:49:21.934350+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:44.504161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033889199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956221+0000","last_change":"2024-09-21T09:49:14.324021+0000","last_active":"2024-09-21T09:49:21.956221+0000","last_peered":"2024-09-21T09:49:21.956221+0000","last_clean":"2024-09-21T09:49:21.956221+0000","last_became_active":"2024-09-21T09:49:10.249765+0000","last_became_peered":"2024-09-21T09:49:10.249765+0000","last_unstale":"2024-09-21T09:49:21.956221+0000","last_undegraded":"2024-09-21T09:49:21.956221+0000","last_fullsized":"2024-09-21T09:49:21.956221+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:13.890155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031296500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.934314+0000","last_change":"2024-09-21T09:49:14.322235+0000","last_active":"2024-09-21T09:49:21.934314+0000","last_peered":"2024-09-21T09:49:21.934314+0000","last_clean":"2024-09-21T09:49:21.934314+0000","last_became_active":"2024-09-21T09:49:10.176921+0000","last_became_peered":"2024-09-21T09:49:10.176921+0000","last_unstale":"2024-09-21T09:49:21.934314+0000","last_undegraded":"2024-09-21T09:49:21.934314+0000","last_fullsized":"2024-09-21T09:49:21.934314+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:51.116299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034704499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956128+0000","last_change":"2024-09-21T09:49:14.321622+0000","last_active":"2024-09-21T09:49:21.956128+0000","last_peered":"2024-09-21T09:49:21.956128+0000","last_clean":"2024-09-21T09:49:21.956128+0000","last_became_active":"2024-09-21T09:49:10.175207+0000","last_became_peered":"2024-09-21T09:49:10.175207+0000","last_unstale":"2024-09-21T09:49:21.956128+0000","last_undegraded":"2024-09-21T09:49:21.956128+0000","last_fullsized":"2024-09-21T09:49:21.956128+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:54.922603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091875599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956079+0000","last_change":"2024-09-21T09:49:14.317504+0000","last_active":"2024-09-21T09:49:21.956079+0000","last_peered":"2024-09-21T09:49:21.956079+0000","last_clean":"2024-09-21T09:49:21.956079+0000","last_became_active":"2024-09-21T09:49:10.249697+0000","last_became_peered":"2024-09-21T09:49:10.249697+0000","last_unstale":"2024-09-21T09:49:21.956079+0000","last_undegraded":"2024-09-21T09:49:21.956079+0000","last_fullsized":"2024-09-21T09:49:21.956079+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:22.915376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039017899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956181+0000","last_change":"2024-09-21T09:49:14.324025+0000","last_active":"2024-09-21T09:49:21.956181+0000","last_peered":"2024-09-21T09:49:21.956181+0000","last_clean":"2024-09-21T09:49:21.956181+0000","last_became_active":"2024-09-21T09:49:10.177038+0000","last_became_peered":"2024-09-21T09:49:10.177038+0000","last_unstale":"2024-09-21T09:49:21.956181+0000","last_undegraded":"2024-09-21T09:49:21.956181+0000","last_fullsized":"2024-09-21T09:49:21.956181+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:22.188993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000343004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:49:21.956150+0000","last_change":"2024-09-21T09:49:14.321731+0000","last_active":"2024-09-21T09:49:21.956150+0000","last_peered":"2024-09-21T09:49:21.956150+0000","last_clean":"2024-09-21T09:49:21.956150+0000","last_became_active":"2024-09-21T09:49:10.174839+0000","last_became_peered":"2024-09-21T09:49:10.174839+0000","last_unstale":"2024-09-21T09:49:21.956150+0000","last_undegraded":"2024-09-21T09:49:21.956150+0000","last_fullsized":"2024-09-21T09:49:21.956150+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:03.713393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010253300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":263,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:49:14.323700+0000","last_change":"2024-09-21T09:48:37.934335+0000","last_active":"2024-09-21T09:49:14.323700+0000","last_peered":"2024-09-21T09:49:14.323700+0000","last_clean":"2024-09-21T09:49:14.323700+0000","last_became_active":"2024-09-21T09:48:37.933834+0000","last_became_peered":"2024-09-21T09:48:37.933834+0000","last_unstale":"2024-09-21T09:49:14.323700+0000","last_undegraded":"2024-09-21T09:49:14.323700+0000","last_fullsized":"2024-09-21T09:49:14.323700+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:09.157244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28252,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716900,"statfs":{"total":95995035648,"available":95966105600,"internally_reserved":0,"allocated":1261568,"data_stored":873130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.59999999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.52200000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.504},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.72099999999999997}]},{"osd":4,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.33600000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.40999999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495643,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":873480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.497}]},{"osd":1,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.51200000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.73399999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.66500000000000004}]},{"osd":4,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.70599999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.747},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.58299999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757280,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":283112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.55700000000000005,"15min":0.55500000000000005},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.502},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38900000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.51200000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.78000000000000003,"15min":0.81000000000000005},"min":{"1min":0.40400000000000003,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.98199999999999998,"5min":7.2190000000000003,"15min":7.2190000000000003},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56200000000000006,"15min":0.56000000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75700000000000001,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.55900000000000005}]},{"osd":2,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.76200000000000001,"15min":0.79600000000000004},"min":{"1min":0.39400000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80500000000000005,"5min":7.1619999999999999,"15min":7.1619999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.38400000000000001,"5min":0.376,"15min":0.376},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.36699999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.72099999999999997,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.81200000000000006,"15min":0.84399999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2130000000000001,"5min":7.181,"15min":7.181},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.441,"15min":0.435},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.46700000000000003,"15min":0.46100000000000002},"min":{"1min":0.316,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.36599999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.35199999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:49:37 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.46300000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018916,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31772,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713380,"statfs":{"total":95995035648,"available":95962501120,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.438,"5min":0.313,"15min":0.313},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.76100000000000001,"15min":0.79700000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78700000000000003,"5min":5.4029999999999996,"15min":5.4029999999999996},"last":0.61799999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55800000000000005,"15min":0.56200000000000006},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.80300000000000005,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.52500000000000002},"min":{"1min":0.41199999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52400000000000002,"15min":0.52500000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.68200000000000005,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.438},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.58799999999999997,"15min":0.58599999999999997},"min":{"1min":0.441,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.56799999999999995}]},{"osd":3,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.86099999999999999,"15min":0.82799999999999996},"min":{"1min":0.45800000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.0060000000000002,"5min":8.0060000000000002,"15min":8.0060000000000002},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.77800000000000002,"15min":0.80600000000000005},"min":{"1min":0.46899999999999997,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.86899999999999999,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.64900000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:49:48 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.40000000000000002,"15min":0.38400000000000001},"min":{"1min":0.309,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.41499999999999998,"15min":0.39500000000000002},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.628},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.60299999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182441,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:07 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.39300000000000002,"15min":0.39500000000000002},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":0.57399999999999995,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.42399999999999999,"15min":0.438},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.44700000000000001,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.251}]},{"osd":1,"last update":"Sat Sep 21 09:50:07 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.443,"15min":0.45500000000000002},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.58799999999999997,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.40500000000000003,"15min":0.41099999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.51200000000000001,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.309}]},{"osd":2,"last update":"Sat Sep 21 09:50:07 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.45400000000000001,"15min":0.46200000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.626,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.44500000000000001,"15min":0.45900000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.54400000000000004,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.41699999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:49:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.59899999999999998,"15min":0.60399999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77900000000000003,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83699999999999997,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.73899999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.61299999999999999,"15min":0.61899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.82899999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76200000000000001,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.70399999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:49:14 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:49:38 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.60899999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31772,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713380,"statfs":{"total":95995035648,"available":95962501120,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:40 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.35999999999999999,"15min":0.35399999999999998},"min":{"1min":0.23899999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.373,"15min":0.36499999999999999},"min":{"1min":0.217,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:49:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.23599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38700000000000001,"15min":0.38100000000000001},"min":{"1min":0.30099999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.52800000000000002,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.318}]},{"osd":3,"last update":"Sat Sep 21 09:50:07 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.47399999999999998,"15min":0.48199999999999998},"min":{"1min":0.316,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:49:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42599999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76100000000000001,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.59499999999999997,"15min":0.59699999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83799999999999997,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.68400000000000005}]},{"osd":5,"last update":"Sat Sep 21 09:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.59399999999999997,"15min":0.59799999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70599999999999996,"5min":0.878,"15min":0.878},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58499999999999996,"15min":0.58799999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.77200000000000002,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Sep 21 09:49:14 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:49:37 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.66400000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":282839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.315},"min":{"1min":0.20799999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.34200000000000003,"15min":0.32600000000000001},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.307}]},{"osd":2,"last update":"Sat Sep 21 09:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.42399999999999999,"15min":0.40999999999999998},"min":{"1min":0.29499999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.39300000000000002,"15min":0.36799999999999999},"min":{"1min":0.27600000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.627,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.32600000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:50:07 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.40400000000000003,"15min":0.39300000000000002},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.57199999999999995,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.432,"15min":0.42699999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.58399999999999996,"5min":0.63,"15min":0.63},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:49:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54400000000000004,"15min":0.54500000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74199999999999999,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.57699999999999996,"15min":0.58299999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.54400000000000004}]},{"osd":6,"last update":"Sat Sep 21 09:49:14 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.69199999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.628}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32352,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712800,"statfs":{"total":95995035648,"available":95961907200,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:49:24 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.34499999999999997,"15min":0.33500000000000002},"min":{"1min":0.23400000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.34799999999999998,"15min":0.33400000000000002},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.34399999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:49:44 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.52200000000000002,"5min":0.622,"15min":0.622},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.435,"5min":0.441,"15min":0.436},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.58999999999999997,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.307}]},{"osd":3,"last update":"Sat Sep 21 09:50:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.433,"15min":0.439},"min":{"1min":0.29499999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.45600000000000002,"15min":0.47199999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.27200000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:49:28 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.60099999999999998,"15min":0.60699999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.63900000000000001,"15min":0.623},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.4210000000000003,"5min":4.4210000000000003,"15min":4.4210000000000003},"last":0.51300000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:49:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59699999999999998,"15min":0.60099999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.63,"15min":0.64100000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.87,"5min":1.254,"15min":1.254},"last":0.46800000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:50:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59099999999999997,"15min":0.59299999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.84499999999999997,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.438},{"interface":"front","average":{"1min":1.1020000000000001,"5min":0.72199999999999998,"15min":0.65900000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":13.663,"5min":13.663,"15min":13.663},"last":0.38100000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:49:35 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.41999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:50:15.777 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:50:15.777 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:50:15.777 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:50:15.778 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:50:16.071 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:15 smithi032 podman[73024]: 2024-09-21 09:50:15.994904234 +0000 UTC m=+1.216648933 container remove 94962fa133e7e3bd2661f937d3c6ded2ac7c8c66bef854d06eb49989b23f12c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:16.072 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:16 smithi032 podman[73024]: 2024-09-21 09:50:16.070500837 +0000 UTC m=+1.292245491 volume remove dd057108f03e0276d1b591d2ef08363d85cb1e291b7410a6155fd30a581996a8 2024-09-21T09:50:16.168 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:16.400 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:16 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:50:16.400 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:16 smithi032 systemd[1]: Stopped Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:50:16.400 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:16 smithi032 systemd[1]: Starting Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:50:17.513 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-mon[29191]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:17.514 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-mon[40061]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:17.514 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 podman[73231]: 2024-09-21 09:50:17.251054541 +0000 UTC m=+0.019394073 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-21T09:50:17.514 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 podman[73231]: 2024-09-21 09:50:17.428833712 +0000 UTC m=+0.197173290 volume create 50096254d7bc7ca0b1d3a99bd9e3fee7ccb297456531dc2dd321f9955f51dd04 2024-09-21T09:50:17.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:17 smithi045 ceph-mon[33087]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:17.841 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 podman[73231]: 2024-09-21 09:50:17.512483561 +0000 UTC m=+0.280823159 container create 75fcd3b3b5dda2756a64e9c0b140c48e554c51bc52923b7c676d4e7aa1e58c92 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:18.094 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 podman[73231]: 2024-09-21 09:50:17.840008497 +0000 UTC m=+0.608348042 container init 75fcd3b3b5dda2756a64e9c0b140c48e554c51bc52923b7c676d4e7aa1e58c92 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:18.094 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 podman[73231]: 2024-09-21 09:50:17.843756952 +0000 UTC m=+0.612096496 container start 75fcd3b3b5dda2756a64e9c0b140c48e554c51bc52923b7c676d4e7aa1e58c92 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:18.094 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:17.859Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:50:18.094 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:17.859Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:50:18.094 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:17.860Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.32 port=9094 2024-09-21T09:50:18.094 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:17.861Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:50:18.094 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:17.887Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:50:18.094 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:17.887Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:50:18.094 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:17.890Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:50:18.095 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:17.890Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:50:18.095 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 bash[73231]: 75fcd3b3b5dda2756a64e9c0b140c48e554c51bc52923b7c676d4e7aa1e58c92 2024-09-21T09:50:18.095 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:17 smithi032 systemd[1]: Started Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:50:18.503 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:50:18.503 INFO:teuthology.orchestra.run.smithi032.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:50:19.066 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:18 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:19.067 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:18 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:19.067 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:18 smithi032 ceph-mon[40061]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:50:19.067 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:18 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/151227556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:50:19.067 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:18 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:19.067 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:18 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:19.067 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:18 smithi032 ceph-mon[29191]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:50:19.067 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:18 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/151227556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:50:19.092 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:50:19.092 INFO:tasks.cephadm:Setup complete, yielding 2024-09-21T09:50:19.092 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T09:50:19.100 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi032.front.sepia.ceph.com 2024-09-21T09:50:19.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- bash -c 'ceph fs volume create foo' 2024-09-21T09:50:19.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:18 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:18 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:18 smithi045 ceph-mon[33087]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:50:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:18 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/151227556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:50:19.311 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:19.957 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:19.862Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000661386s 2024-09-21T09:50:20.159 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:19 smithi045 ceph-mon[33087]: Reconfiguring daemon prometheus.a on smithi045 2024-09-21T09:50:20.159 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:19 smithi045 ceph-mon[33087]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:20.248 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:19 smithi032 ceph-mon[40061]: Reconfiguring daemon prometheus.a on smithi045 2024-09-21T09:50:20.248 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:19 smithi032 ceph-mon[40061]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:20.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:19 smithi032 ceph-mon[29191]: Reconfiguring daemon prometheus.a on smithi045 2024-09-21T09:50:20.249 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:19 smithi032 ceph-mon[29191]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:21.285 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 systemd[1]: Stopping Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:50:21.647 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-mon[33087]: from='client.14637 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:21.647 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:50:21.647 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-mon[33087]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:21.647 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.369Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:50:21.647 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:50:21.647 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[59508]: ts=2024-09-21T09:50:21.370Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:50:21.648 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 podman[62608]: 2024-09-21 09:50:21.38014718 +0000 UTC m=+0.123704050 container died 6d18867ea721b1573a8facb1b3b0e80f3b40d36e62b8bad907b14e0bb30b2449 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:21 smithi032 ceph-mon[29191]: from='client.14637 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:21 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:50:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:21 smithi032 ceph-mon[29191]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:21 smithi032 ceph-mon[40061]: from='client.14637 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:21 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:50:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:21 smithi032 ceph-mon[40061]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:22.034 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 podman[62608]: 2024-09-21 09:50:21.753228194 +0000 UTC m=+0.496785064 container cleanup 6d18867ea721b1573a8facb1b3b0e80f3b40d36e62b8bad907b14e0bb30b2449 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:22.034 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:21 smithi045 bash[62608]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:50:22.394 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 podman[62620]: 2024-09-21 09:50:22.038045361 +0000 UTC m=+0.659888694 container remove 6d18867ea721b1573a8facb1b3b0e80f3b40d36e62b8bad907b14e0bb30b2449 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:22.394 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:50:22.394 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 systemd[1]: Stopped Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:50:22.394 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.118s CPU time. 2024-09-21T09:50:22.394 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 systemd[1]: Starting Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:50:22.654 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-21T09:50:22.654 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-mon[33087]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:50:22.654 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:50:22.655 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:22.655 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 podman[62711]: 2024-09-21 09:50:22.296270271 +0000 UTC m=+0.017191371 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:50:22.655 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 podman[62711]: 2024-09-21 09:50:22.422620346 +0000 UTC m=+0.143541436 container create b12ebc2bf34d975d6dec270fc7c334bc4beb06e902398f9d6e038aaf88f06ee3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:22.655 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 podman[62711]: 2024-09-21 09:50:22.625049008 +0000 UTC m=+0.345970117 container init b12ebc2bf34d975d6dec270fc7c334bc4beb06e902398f9d6e038aaf88f06ee3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:22.655 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 podman[62711]: 2024-09-21 09:50:22.627830514 +0000 UTC m=+0.348751605 container start b12ebc2bf34d975d6dec270fc7c334bc4beb06e902398f9d6e038aaf88f06ee3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:50:22.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-21T09:50:22.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-mon[40061]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:50:22.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:50:22.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[29187]: 2024-09-21T09:50:22.389+0000 7fd68f32f640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:50:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-21T09:50:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-mon[29191]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:50:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:50:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:22.985 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:22] ENGINE Bus STOPPING 2024-09-21T09:50:22.985 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:50:22.985 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:22] ENGINE Bus STOPPED 2024-09-21T09:50:22.986 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:22] ENGINE Bus STARTING 2024-09-21T09:50:22.986 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:22] ENGINE Serving on http://:::9283 2024-09-21T09:50:22.986 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:22] ENGINE Bus STARTED 2024-09-21T09:50:23.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.653Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:50:23.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.653Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:50:23.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.653Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi045 (none))" 2024-09-21T09:50:23.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.653Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:50:23.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.653Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:50:23.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.655Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.655Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.656Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.656Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.659Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.659Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.852µs 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.659Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.660Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.660Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.660Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.821µs wal_replay_duration=1.504573ms wbl_replay_duration=132ns total_replay_duration=1.557271ms 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.661Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.661Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.661Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.684Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.312182ms db_storage=787ns remote_storage=1.292µs web_handler=477ns query_engine=701ns scrape=447.716µs scrape_sd=99.501µs notify=17.996µs notify_sd=276.796µs rules=21.117355ms tracing=6.31µs 2024-09-21T09:50:23.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.684Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:50:23.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:50:22.684Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:50:23.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 bash[62711]: b12ebc2bf34d975d6dec270fc7c334bc4beb06e902398f9d6e038aaf88f06ee3 2024-09-21T09:50:23.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:50:22 smithi045 systemd[1]: Started Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:50:23.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-21T09:50:23.394 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:22 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:22] ENGINE Bus STOPPING 2024-09-21T09:50:23.631 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-21T09:50:23.631 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:50:23.631 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:50:23.631 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:50:23.631 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T09:50:23.631 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:50:23.631 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-21T09:50:23.631 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: fsmap foo:0 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: Saving service mds.foo spec with placement count:2 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi032.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi032.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.632 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:50:23.633 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:50:23.633 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:50:23.633 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:50:23.633 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.633 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:23.633 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:23 smithi045 ceph-mon[33087]: pgmap v51: 73 pgs: 20 creating+peering, 53 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:23.683 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:50:23.683 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:23] ENGINE Bus STOPPED 2024-09-21T09:50:23.683 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:23] ENGINE Bus STARTING 2024-09-21T09:50:23.683 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:23] ENGINE Serving on http://:::9283 2024-09-21T09:50:23.683 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:23] ENGINE Bus STARTED 2024-09-21T09:50:23.683 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:23] ENGINE Bus STOPPING 2024-09-21T09:50:23.683 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: fsmap foo:0 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: Saving service mds.foo spec with placement count:2 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi032.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:50:23.684 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi032.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:23.685 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[29191]: pgmap v51: 73 pgs: 20 creating+peering, 53 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: fsmap foo:0 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: Saving service mds.foo spec with placement count:2 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:50:23.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi032.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi032.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:23.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:23.688 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:23 smithi032 ceph-mon[40061]: pgmap v51: 73 pgs: 20 creating+peering, 53 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:23.983 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:24.405 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:50:24.405 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:24] ENGINE Bus STOPPED 2024-09-21T09:50:24.405 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:24] ENGINE Bus STARTING 2024-09-21T09:50:24.405 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:24] ENGINE Serving on http://:::9283 2024-09-21T09:50:24.405 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:50:24] ENGINE Bus STARTED 2024-09-21T09:50:24.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:24 smithi032 ceph-mon[29191]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:50:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:24 smithi032 ceph-mon[40061]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:50:24.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:24 smithi045 ceph-mon[33087]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:50:25.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:25 smithi032 ceph-mon[40061]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:50:25.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:25 smithi032 ceph-mon[40061]: pgmap v54: 73 pgs: 20 creating+peering, 53 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:25.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:25 smithi032 ceph-mon[29191]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:50:25.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:25 smithi032 ceph-mon[29191]: pgmap v54: 73 pgs: 20 creating+peering, 53 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:25.707 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:25 smithi045 ceph-mon[33087]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:50:25.707 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:25 smithi045 ceph-mon[33087]: pgmap v54: 73 pgs: 20 creating+peering, 53 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:26.366 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-21T09:50:27.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:27 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:27.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:27 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:27.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:27 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:27.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:27 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:27.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:27 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:27 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:27.947 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:28.166 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:50:27 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:50:27.864Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003074829s 2024-09-21T09:50:28.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[40061]: pgmap v55: 73 pgs: 20 creating+peering, 53 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:28.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:28.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:28.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:28.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:28.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:28.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi032.fdolcm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:50:28.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi032.fdolcm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:50:28.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:28.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[29191]: pgmap v55: 73 pgs: 20 creating+peering, 53 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:28.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:28.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:28.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:28.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:28.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:28.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi032.fdolcm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:50:28.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi032.fdolcm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:50:28.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:28 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:28 smithi045 ceph-mon[33087]: pgmap v55: 73 pgs: 20 creating+peering, 53 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:28 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:28 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:28 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:28 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:28 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:28 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi032.fdolcm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:50:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:28 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi032.fdolcm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:50:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:28 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:29.424 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:29 smithi032 ceph-mon[29191]: Deploying daemon mds.foo.smithi032.fdolcm on smithi032 2024-09-21T09:50:29.424 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:29 smithi032 ceph-mon[40061]: Deploying daemon mds.foo.smithi032.fdolcm on smithi032 2024-09-21T09:50:29.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:29 smithi045 ceph-mon[33087]: Deploying daemon mds.foo.smithi032.fdolcm on smithi032 2024-09-21T09:50:30.426 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:30 smithi032 ceph-mon[40061]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:30.426 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:30 smithi032 ceph-mon[29191]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:30.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:30 smithi045 ceph-mon[33087]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:31.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:31 smithi032 ceph-mon[29191]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:31.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:31 smithi032 ceph-mon[40061]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:31.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:31 smithi045 ceph-mon[33087]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:31.602 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T09:50:31.612 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:50:31.612 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:50:31.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:50:32.199 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:33.426 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: mds.? [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] up:boot 2024-09-21T09:50:33.426 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: daemon mds.foo.smithi032.fdolcm assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: Cluster is now healthy 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: fsmap foo:0 1 up:standby 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:creating} 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: daemon mds.foo.smithi032.fdolcm is now active in filesystem foo as rank 0 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi045.jnfsfb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi045.jnfsfb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:50:33.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: Deploying daemon mds.foo.smithi045.jnfsfb on smithi045 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[40061]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: mds.? [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] up:boot 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: daemon mds.foo.smithi032.fdolcm assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: Cluster is now healthy 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: fsmap foo:0 1 up:standby 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:creating} 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: daemon mds.foo.smithi032.fdolcm is now active in filesystem foo as rank 0 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:33.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:33.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:33.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi045.jnfsfb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:50:33.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi045.jnfsfb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:50:33.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:33.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: Deploying daemon mds.foo.smithi045.jnfsfb on smithi045 2024-09-21T09:50:33.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-mon[29191]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-21T09:50:33.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: mds.? [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] up:boot 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: daemon mds.foo.smithi032.fdolcm assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: Cluster is now healthy 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: fsmap foo:0 1 up:standby 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:creating} 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: daemon mds.foo.smithi032.fdolcm is now active in filesystem foo as rank 0 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi045.jnfsfb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:50:33.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi045.jnfsfb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:50:33.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:33.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: Deploying daemon mds.foo.smithi045.jnfsfb on smithi045 2024-09-21T09:50:33.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:33 smithi045 ceph-mon[33087]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-21T09:50:33.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:33 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:50:33] "GET /metrics HTTP/1.1" 200 30788 "" "Prometheus/2.43.0" 2024-09-21T09:50:33.876 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:50:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:34 smithi045 ceph-mon[33087]: mds.? [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] up:active 2024-09-21T09:50:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:34 smithi045 ceph-mon[33087]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 2024-09-21T09:50:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:34 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1912374517' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:50:34.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:34 smithi032 ceph-mon[29191]: mds.? [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] up:active 2024-09-21T09:50:34.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:34 smithi032 ceph-mon[29191]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 2024-09-21T09:50:34.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:34 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1912374517' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:50:34.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:34 smithi032 ceph-mon[40061]: mds.? [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] up:active 2024-09-21T09:50:34.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:34 smithi032 ceph-mon[40061]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 2024-09-21T09:50:34.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:34 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1912374517' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:50:34.663 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":20,"flags":0,"active_gid":24335,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":431176591},{"type":"v1","addr":"172.21.15.32:6801","nonce":431176591}]},"active_addr":"172.21.15.32:6801/431176591","active_change":"2024-09-21T09:49:07.126081+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24346,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.32:8443/","prometheus":"http://172.21.15.32:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1200491506}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1465894634}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1858508442}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":2103196055}]}]} 2024-09-21T09:50:34.667 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:50:34.667 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:50:34.667 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:50:34.887 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: mds.? [v2:172.21.15.45:6832/3743591142,v1:172.21.15.45:6833/3743591142] up:boot 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:50:36.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:35 smithi045 ceph-mon[33087]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: mds.? [v2:172.21.15.45:6832/3743591142,v1:172.21.15.45:6833/3743591142] up:boot 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:36.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:50:36.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[29191]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:36.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:36.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:36.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:50:36.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: mds.? [v2:172.21.15.45:6832/3743591142,v1:172.21.15.45:6833/3743591142] up:boot 2024-09-21T09:50:36.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:36.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:50:36.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:35 smithi032 ceph-mon[40061]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:36.838 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:50:36.839 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":71,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:43:37.317307+0000","modified":"2024-09-21T09:50:24.404985+0000","last_up_change":"2024-09-21T09:48:34.864213+0000","last_in_change":"2024-09-21T09:48:18.335792+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:46:40.328691+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:49:08.740859+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:50:21.329505+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:50:21.832696+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0f6f5dda-c134-4283-a91d-cf501ef19062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6803","nonce":2765215923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6805","nonce":2765215923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6809","nonce":2765215923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6807","nonce":2765215923}]},"public_addr":"172.21.15.32:6803/2765215923","cluster_addr":"172.21.15.32:6805/2765215923","heartbeat_back_addr":"172.21.15.32:6809/2765215923","heartbeat_front_addr":"172.21.15.32:6807/2765215923","state":["exists","up"]},{"osd":1,"uuid":"bdaa43fd-b969-4d06-b6eb-97b415c4e118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6811","nonce":3808018383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6813","nonce":3808018383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6817","nonce":3808018383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6815","nonce":3808018383}]},"public_addr":"172.21.15.32:6811/3808018383","cluster_addr":"172.21.15.32:6813/3808018383","heartbeat_back_addr":"172.21.15.32:6817/3808018383","heartbeat_front_addr":"172.21.15.32:6815/3808018383","state":["exists","up"]},{"osd":2,"uuid":"1f12d9f2-90b0-44f6-8432-0b398194b3b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6819","nonce":2468805298}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6821","nonce":2468805298}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6825","nonce":2468805298}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6823","nonce":2468805298}]},"public_addr":"172.21.15.32:6819/2468805298","cluster_addr":"172.21.15.32:6821/2468805298","heartbeat_back_addr":"172.21.15.32:6825/2468805298","heartbeat_front_addr":"172.21.15.32:6823/2468805298","state":["exists","up"]},{"osd":3,"uuid":"6a568fd5-5fd9-46f8-b3af-9fa196ed617b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6827","nonce":2765373470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6829","nonce":2765373470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6833","nonce":2765373470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6831","nonce":2765373470}]},"public_addr":"172.21.15.32:6827/2765373470","cluster_addr":"172.21.15.32:6829/2765373470","heartbeat_back_addr":"172.21.15.32:6833/2765373470","heartbeat_front_addr":"172.21.15.32:6831/2765373470","state":["exists","up"]},{"osd":4,"uuid":"2cf149e0-4c2f-46eb-9ca2-a51907aea3b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6801","nonce":413300252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6803","nonce":413300252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6807","nonce":413300252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6805","nonce":413300252}]},"public_addr":"172.21.15.45:6801/413300252","cluster_addr":"172.21.15.45:6803/413300252","heartbeat_back_addr":"172.21.15.45:6807/413300252","heartbeat_front_addr":"172.21.15.45:6805/413300252","state":["exists","up"]},{"osd":5,"uuid":"a6c78855-6756-4b0b-9a9c-8d102079540b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6809","nonce":3168849729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6811","nonce":3168849729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6815","nonce":3168849729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6813","nonce":3168849729}]},"public_addr":"172.21.15.45:6809/3168849729","cluster_addr":"172.21.15.45:6811/3168849729","heartbeat_back_addr":"172.21.15.45:6815/3168849729","heartbeat_front_addr":"172.21.15.45:6813/3168849729","state":["exists","up"]},{"osd":6,"uuid":"d476169d-dffc-4f1e-b60b-4f836f53285e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6817","nonce":2158139679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6819","nonce":2158139679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6823","nonce":2158139679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6821","nonce":2158139679}]},"public_addr":"172.21.15.45:6817/2158139679","cluster_addr":"172.21.15.45:6819/2158139679","heartbeat_back_addr":"172.21.15.45:6823/2158139679","heartbeat_front_addr":"172.21.15.45:6821/2158139679","state":["exists","up"]},{"osd":7,"uuid":"021a4fd0-f770-4799-9683-b0aa7cd593bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6825","nonce":3336053350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6827","nonce":3336053350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6831","nonce":3336053350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6829","nonce":3336053350}]},"public_addr":"172.21.15.45:6825/3336053350","cluster_addr":"172.21.15.45:6827/3336053350","heartbeat_back_addr":"172.21.15.45:6831/3336053350","heartbeat_front_addr":"172.21.15.45:6829/3336053350","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:50.697928+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:13.831263+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:37.360357+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:00.010474+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:21.607129+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:45.583237+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:09.486296+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:33.456690+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6800/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/1259145858":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/230160407":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/3332221400":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/1178553739":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/458122323":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/2895073807":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6801/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/533058733":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2042031504":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2388392475":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/2785183325":"2024-09-22T09:49:07.125921+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:50:37.538 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:50:37.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:50:37.883 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:37.996 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:37 smithi045 ceph-mon[33087]: mds.? [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] up:active 2024-09-21T09:50:37.996 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:37 smithi045 ceph-mon[33087]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:37.996 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:37 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/2483290997' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:50:37.996 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:37 smithi045 ceph-mon[33087]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:37.996 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:37 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:37.996 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:37 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:38.138 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[29191]: mds.? [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] up:active 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[29191]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2483290997' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[29191]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[40061]: mds.? [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] up:active 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[40061]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/2483290997' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[40061]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:38.139 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:37 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:39.323 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:39 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:39.323 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:39 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:39.323 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:39 smithi032 ceph-mon[29191]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:39.323 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:39 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:39.323 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:39 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:39.323 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:39 smithi032 ceph-mon[40061]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:39 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:39 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:39 smithi045 ceph-mon[33087]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:39.726 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:50:39.726 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":71,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:43:37.317307+0000","modified":"2024-09-21T09:50:24.404985+0000","last_up_change":"2024-09-21T09:48:34.864213+0000","last_in_change":"2024-09-21T09:48:18.335792+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:46:40.328691+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:49:08.740859+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:50:21.329505+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:50:21.832696+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0f6f5dda-c134-4283-a91d-cf501ef19062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6803","nonce":2765215923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6805","nonce":2765215923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6809","nonce":2765215923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2765215923},{"type":"v1","addr":"172.21.15.32:6807","nonce":2765215923}]},"public_addr":"172.21.15.32:6803/2765215923","cluster_addr":"172.21.15.32:6805/2765215923","heartbeat_back_addr":"172.21.15.32:6809/2765215923","heartbeat_front_addr":"172.21.15.32:6807/2765215923","state":["exists","up"]},{"osd":1,"uuid":"bdaa43fd-b969-4d06-b6eb-97b415c4e118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6811","nonce":3808018383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6813","nonce":3808018383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6817","nonce":3808018383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":3808018383},{"type":"v1","addr":"172.21.15.32:6815","nonce":3808018383}]},"public_addr":"172.21.15.32:6811/3808018383","cluster_addr":"172.21.15.32:6813/3808018383","heartbeat_back_addr":"172.21.15.32:6817/3808018383","heartbeat_front_addr":"172.21.15.32:6815/3808018383","state":["exists","up"]},{"osd":2,"uuid":"1f12d9f2-90b0-44f6-8432-0b398194b3b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6819","nonce":2468805298}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6821","nonce":2468805298}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6825","nonce":2468805298}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":2468805298},{"type":"v1","addr":"172.21.15.32:6823","nonce":2468805298}]},"public_addr":"172.21.15.32:6819/2468805298","cluster_addr":"172.21.15.32:6821/2468805298","heartbeat_back_addr":"172.21.15.32:6825/2468805298","heartbeat_front_addr":"172.21.15.32:6823/2468805298","state":["exists","up"]},{"osd":3,"uuid":"6a568fd5-5fd9-46f8-b3af-9fa196ed617b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6827","nonce":2765373470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6829","nonce":2765373470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6833","nonce":2765373470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":2765373470},{"type":"v1","addr":"172.21.15.32:6831","nonce":2765373470}]},"public_addr":"172.21.15.32:6827/2765373470","cluster_addr":"172.21.15.32:6829/2765373470","heartbeat_back_addr":"172.21.15.32:6833/2765373470","heartbeat_front_addr":"172.21.15.32:6831/2765373470","state":["exists","up"]},{"osd":4,"uuid":"2cf149e0-4c2f-46eb-9ca2-a51907aea3b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6801","nonce":413300252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6803","nonce":413300252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6807","nonce":413300252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6805","nonce":413300252}]},"public_addr":"172.21.15.45:6801/413300252","cluster_addr":"172.21.15.45:6803/413300252","heartbeat_back_addr":"172.21.15.45:6807/413300252","heartbeat_front_addr":"172.21.15.45:6805/413300252","state":["exists","up"]},{"osd":5,"uuid":"a6c78855-6756-4b0b-9a9c-8d102079540b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6809","nonce":3168849729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6811","nonce":3168849729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6815","nonce":3168849729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6813","nonce":3168849729}]},"public_addr":"172.21.15.45:6809/3168849729","cluster_addr":"172.21.15.45:6811/3168849729","heartbeat_back_addr":"172.21.15.45:6815/3168849729","heartbeat_front_addr":"172.21.15.45:6813/3168849729","state":["exists","up"]},{"osd":6,"uuid":"d476169d-dffc-4f1e-b60b-4f836f53285e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6817","nonce":2158139679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6819","nonce":2158139679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6823","nonce":2158139679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6821","nonce":2158139679}]},"public_addr":"172.21.15.45:6817/2158139679","cluster_addr":"172.21.15.45:6819/2158139679","heartbeat_back_addr":"172.21.15.45:6823/2158139679","heartbeat_front_addr":"172.21.15.45:6821/2158139679","state":["exists","up"]},{"osd":7,"uuid":"021a4fd0-f770-4799-9683-b0aa7cd593bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6825","nonce":3336053350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6827","nonce":3336053350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6831","nonce":3336053350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6829","nonce":3336053350}]},"public_addr":"172.21.15.45:6825/3336053350","cluster_addr":"172.21.15.45:6827/3336053350","heartbeat_back_addr":"172.21.15.45:6831/3336053350","heartbeat_front_addr":"172.21.15.45:6829/3336053350","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:50.697928+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:13.831263+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:37.360357+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:00.010474+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:21.607129+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:45.583237+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:09.486296+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:33.456690+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:6801/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6800/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/1259145858":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/230160407":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/3332221400":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/1178553739":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/458122323":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/2895073807":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6801/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/533058733":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2042031504":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2388392475":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:0/2785183325":"2024-09-22T09:49:07.125921+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:50:40.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:40 smithi045 ceph-mon[33087]: mds.? [v2:172.21.15.45:6832/3743591142,v1:172.21.15.45:6833/3743591142] up:standby 2024-09-21T09:50:40.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:40 smithi045 ceph-mon[33087]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:40.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:40 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/530017460' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:50:40.644 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:40 smithi032 ceph-mon[40061]: mds.? [v2:172.21.15.45:6832/3743591142,v1:172.21.15.45:6833/3743591142] up:standby 2024-09-21T09:50:40.644 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:40 smithi032 ceph-mon[40061]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:40.644 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:40 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/530017460' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:50:40.644 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:40 smithi032 ceph-mon[29191]: mds.? [v2:172.21.15.45:6832/3743591142,v1:172.21.15.45:6833/3743591142] up:standby 2024-09-21T09:50:40.644 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:40 smithi032 ceph-mon[29191]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:50:40.644 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:40 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/530017460' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:50:40.672 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:50:40.673 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:50:40.673 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:50:40.673 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:50:40.674 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:50:40.674 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:50:40.674 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:50:40.675 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:50:41.127 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:41.177 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:41.346 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:41 smithi032 ceph-mon[29191]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:41.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:41 smithi032 ceph-mon[40061]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:41.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:41 smithi045 ceph-mon[33087]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:43.025 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:43.025 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:43.025 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:43.029 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:43.029 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:43.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:43 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:43 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:43 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:43 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:43 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:43 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:43 smithi045 ceph-mon[33087]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:43.626 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:43.626 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:43.626 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:43.626 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:43.626 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:43.626 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:43.626 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[40061]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:43.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:43.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:43.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:43.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:43.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:43.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:43.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-mon[29191]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:50:43.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:43 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:50:43] "GET /metrics HTTP/1.1" 200 33691 "" "Prometheus/2.43.0" 2024-09-21T09:50:45.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:45 smithi045 ceph-mon[33087]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:50:45.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:45 smithi032 ceph-mon[29191]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:50:45.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:45 smithi032 ceph-mon[40061]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:50:46.775 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:46 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:46.776 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:46 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:46.776 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:46 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:46.776 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:46 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:46.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:46 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:46.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:46 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:50:47.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:47 smithi032 ceph-mon[29191]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:50:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:47 smithi032 ceph-mon[40061]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:50:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:47 smithi045 ceph-mon[33087]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:50:49.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:49 smithi032 ceph-mon[29191]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:50:49.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:49 smithi032 ceph-mon[40061]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:50:49.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:49 smithi045 ceph-mon[33087]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:50:49.660 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:51.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:51 smithi032 ceph-mon[29191]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:51.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:51 smithi032 ceph-mon[40061]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:51.490 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:51 smithi045 ceph-mon[33087]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:51.636 INFO:teuthology.orchestra.run.smithi032.stdout:128849018923 2024-09-21T09:50:51.637 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:50:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:52 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:52 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:52.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:52 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:52.539 INFO:teuthology.orchestra.run.smithi032.stdout:231928234013 2024-09-21T09:50:52.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:50:53.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:53 smithi032 ceph-mon[40061]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:53.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:53 smithi032 ceph-mon[29191]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:53.346 INFO:teuthology.orchestra.run.smithi032.stdout:163208757288 2024-09-21T09:50:53.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:50:53.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:53 smithi045 ceph-mon[33087]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:53.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:50:53 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:50:53] "GET /metrics HTTP/1.1" 200 33692 "" "Prometheus/2.43.0" 2024-09-21T09:50:55.422 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:55 smithi032 ceph-mon[40061]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:55.422 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:55 smithi032 ceph-mon[29191]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:55.450 INFO:teuthology.orchestra.run.smithi032.stdout:77309411382 2024-09-21T09:50:55.450 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:50:55.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:55 smithi045 ceph-mon[33087]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:56.202 INFO:teuthology.orchestra.run.smithi032.stdout:34359738432 2024-09-21T09:50:56.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:50:57.395 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:57 smithi032 ceph-mon[40061]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:57.395 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:57 smithi032 ceph-mon[29191]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:57.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:57 smithi045 ceph-mon[33087]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:58.236 INFO:teuthology.orchestra.run.smithi032.stdout:107374182450 2024-09-21T09:50:58.236 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:50:59.347 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:50:59 smithi032 ceph-mon[40061]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:59.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:50:59 smithi032 ceph-mon[29191]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:59.368 INFO:teuthology.orchestra.run.smithi032.stdout:55834574909 2024-09-21T09:50:59.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:50:59.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:50:59 smithi045 ceph-mon[33087]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:50:59.709 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:59.713 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:59.747 INFO:teuthology.orchestra.run.smithi032.stdout:197568495652 2024-09-21T09:50:59.747 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:50:59.948 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:59.972 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:01.261 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:01 smithi032 ceph-mon[40061]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:01.262 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:01 smithi032 ceph-mon[29191]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:01.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:01 smithi045 ceph-mon[33087]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:03.439 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:03 smithi032 ceph-mon[40061]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:03.439 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:03 smithi032 ceph-mon[29191]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:03.475 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:03.475 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:03.476 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:03.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:03 smithi045 ceph-mon[33087]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:03.695 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:51:03] "GET /metrics HTTP/1.1" 200 33692 "" "Prometheus/2.43.0" 2024-09-21T09:51:03.702 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:04.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:04 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3377678060' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:51:04.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:04 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3377678060' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:51:04.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:04 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3377678060' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:51:05.672 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:05 smithi032 ceph-mon[29191]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:05.673 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:05 smithi032 ceph-mon[40061]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:05.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:05 smithi045 ceph-mon[33087]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:07.258 INFO:teuthology.orchestra.run.smithi032.stdout:107374182452 2024-09-21T09:51:07.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-21T09:51:07.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-21T09:51:07.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:51:07.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[40061]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:07.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-21T09:51:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-21T09:51:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:51:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[29191]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:07.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:07.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-21T09:51:07.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-21T09:51:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:51:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:07 smithi045 ceph-mon[33087]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:08.082 INFO:teuthology.orchestra.run.smithi032.stdout:128849018927 2024-09-21T09:51:08.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[29191]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/179846180' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1309202721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[29191]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[40061]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:51:08.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/179846180' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:51:08.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1309202721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:51:08.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:08 smithi032 ceph-mon[40061]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:51:08.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:08 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-21T09:51:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:08 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-21T09:51:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:08 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T09:51:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:08 smithi045 ceph-mon[33087]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:51:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:08 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/179846180' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:51:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:08 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1309202721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:51:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:08 smithi045 ceph-mon[33087]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:51:08.660 INFO:teuthology.orchestra.run.smithi032.stdout:231928234016 2024-09-21T09:51:08.759 INFO:teuthology.orchestra.run.smithi032.stdout:77309411384 2024-09-21T09:51:09.424 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:09 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3721585502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:51:09.424 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:09 smithi032 ceph-mon[29191]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:09.425 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:09 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3721585502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:51:09.425 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:09 smithi032 ceph-mon[40061]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:09 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3721585502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:51:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:09 smithi045 ceph-mon[33087]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:09.836 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018923 got 128849018927 for osd.4 2024-09-21T09:51:09.836 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:10.955 INFO:teuthology.orchestra.run.smithi032.stdout:197568495655 2024-09-21T09:51:11.052 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:11 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1476967605' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:51:11.053 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:11 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1476967605' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:51:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:11 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1476967605' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:51:11.321 INFO:teuthology.orchestra.run.smithi032.stdout:34359738435 2024-09-21T09:51:12.266 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:12 smithi032 ceph-mon[29191]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:12.266 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:12 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1711085039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:51:12.267 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:12 smithi032 ceph-mon[40061]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:12.267 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:12 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1711085039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:51:12.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:12 smithi045 ceph-mon[33087]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:12.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:12 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1711085039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:51:12.701 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234013 got 231928234016 for osd.7 2024-09-21T09:51:12.702 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:13.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:13 smithi032 ceph-mon[29191]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:13.430 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:13 smithi032 ceph-mon[40061]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:13.456 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411382 got 77309411384 for osd.2 2024-09-21T09:51:13.456 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:13.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:13 smithi045 ceph-mon[33087]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:13.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:51:13] "GET /metrics HTTP/1.1" 200 33690 "" "Prometheus/2.43.0" 2024-09-21T09:51:13.817 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182450 got 107374182452 for osd.3 2024-09-21T09:51:13.817 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:14.680 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738432 got 34359738435 for osd.0 2024-09-21T09:51:14.681 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:15.509 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:15 smithi032 ceph-mon[40061]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:15.510 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:15 smithi032 ceph-mon[29191]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:15.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:15 smithi045 ceph-mon[33087]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:15.870 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495652 got 197568495655 for osd.6 2024-09-21T09:51:15.870 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:16.250 INFO:teuthology.orchestra.run.smithi032.stdout:163208757293 2024-09-21T09:51:16.629 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:16 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1674144797' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:51:16.630 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:16 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1674144797' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:51:16.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:16 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1674144797' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:51:17.026 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757288 got 163208757293 for osd.5 2024-09-21T09:51:17.027 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:17.420 INFO:teuthology.orchestra.run.smithi032.stdout:55834574912 2024-09-21T09:51:17.557 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:17 smithi032 ceph-mon[40061]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:17.558 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:17 smithi032 ceph-mon[29191]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:17.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:17 smithi045 ceph-mon[33087]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:18.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:18 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3945803602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:51:18.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:18 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3945803602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:51:18.760 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574909 got 55834574912 for osd.1 2024-09-21T09:51:18.760 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:18.760 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:51:18.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:51:18.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:18 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3945803602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:51:18.986 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:20.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:20.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:20.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[29191]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:20.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:20.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:20.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:20.134 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:20.135 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:20.135 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[40061]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:20.135 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:20.135 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:20.135 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:19 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:20.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:19 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:19 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:19 smithi045 ceph-mon[33087]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:19 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:19 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:19 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:20.826 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:51:20.827 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T09:51:21.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:21 smithi032 ceph-mon[29191]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:51:21.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:21 smithi032 ceph-mon[29191]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:21.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:21 smithi032 ceph-mon[40061]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:51:21.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:21 smithi032 ceph-mon[40061]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:21.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:21 smithi045 ceph-mon[33087]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:51:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:21 smithi045 ceph-mon[33087]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:21.594 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":83,"stamp":"2024-09-21T09:51:19.164228+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":236232,"kb_used_data":7760,"kb_used_omap":12,"kb_used_meta":228339,"kb_avail":749724984,"statfs":{"total":767960285184,"available":767718383616,"internally_reserved":0,"allocated":7946240,"data_stored":4446475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":233819704},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.961262"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204424+0000","last_change":"2024-09-21T09:50:22.930135+0000","last_active":"2024-09-21T09:51:07.204424+0000","last_peered":"2024-09-21T09:51:07.204424+0000","last_clean":"2024-09-21T09:51:07.204424+0000","last_became_active":"2024-09-21T09:50:22.929856+0000","last_became_peered":"2024-09-21T09:50:22.929856+0000","last_unstale":"2024-09-21T09:51:07.204424+0000","last_undegraded":"2024-09-21T09:51:07.204424+0000","last_fullsized":"2024-09-21T09:51:07.204424+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:02.058108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635768+0000","last_change":"2024-09-21T09:50:22.400322+0000","last_active":"2024-09-21T09:51:07.635768+0000","last_peered":"2024-09-21T09:51:07.635768+0000","last_clean":"2024-09-21T09:51:07.635768+0000","last_became_active":"2024-09-21T09:50:22.400017+0000","last_became_peered":"2024-09-21T09:50:22.400017+0000","last_unstale":"2024-09-21T09:51:07.635768+0000","last_undegraded":"2024-09-21T09:51:07.635768+0000","last_fullsized":"2024-09-21T09:51:07.635768+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:24:27.572098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209060+0000","last_change":"2024-09-21T09:50:22.931558+0000","last_active":"2024-09-21T09:51:07.209060+0000","last_peered":"2024-09-21T09:51:07.209060+0000","last_clean":"2024-09-21T09:51:07.209060+0000","last_became_active":"2024-09-21T09:50:22.931278+0000","last_became_peered":"2024-09-21T09:50:22.931278+0000","last_unstale":"2024-09-21T09:51:07.209060+0000","last_undegraded":"2024-09-21T09:51:07.209060+0000","last_fullsized":"2024-09-21T09:51:07.209060+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:12.076005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635285+0000","last_change":"2024-09-21T09:50:22.407053+0000","last_active":"2024-09-21T09:51:07.635285+0000","last_peered":"2024-09-21T09:51:07.635285+0000","last_clean":"2024-09-21T09:51:07.635285+0000","last_became_active":"2024-09-21T09:50:22.406593+0000","last_became_peered":"2024-09-21T09:50:22.406593+0000","last_unstale":"2024-09-21T09:51:07.635285+0000","last_undegraded":"2024-09-21T09:51:07.635285+0000","last_fullsized":"2024-09-21T09:51:07.635285+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:50.832120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635514+0000","last_change":"2024-09-21T09:50:23.416113+0000","last_active":"2024-09-21T09:51:07.635514+0000","last_peered":"2024-09-21T09:51:07.635514+0000","last_clean":"2024-09-21T09:51:07.635514+0000","last_became_active":"2024-09-21T09:50:23.415818+0000","last_became_peered":"2024-09-21T09:50:23.415818+0000","last_unstale":"2024-09-21T09:51:07.635514+0000","last_undegraded":"2024-09-21T09:51:07.635514+0000","last_fullsized":"2024-09-21T09:51:07.635514+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:26.685970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635721+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T09:51:07.635721+0000","last_peered":"2024-09-21T09:51:07.635721+0000","last_clean":"2024-09-21T09:51:07.635721+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T09:51:07.635721+0000","last_undegraded":"2024-09-21T09:51:07.635721+0000","last_fullsized":"2024-09-21T09:51:07.635721+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203808+0000","last_change":"2024-09-21T09:50:22.930261+0000","last_active":"2024-09-21T09:51:07.203808+0000","last_peered":"2024-09-21T09:51:07.203808+0000","last_clean":"2024-09-21T09:51:07.203808+0000","last_became_active":"2024-09-21T09:50:22.929920+0000","last_became_peered":"2024-09-21T09:50:22.929920+0000","last_unstale":"2024-09-21T09:51:07.203808+0000","last_undegraded":"2024-09-21T09:51:07.203808+0000","last_fullsized":"2024-09-21T09:51:07.203808+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:50.274508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.206148+0000","last_change":"2024-09-21T09:50:22.398833+0000","last_active":"2024-09-21T09:51:07.206148+0000","last_peered":"2024-09-21T09:51:07.206148+0000","last_clean":"2024-09-21T09:51:07.206148+0000","last_became_active":"2024-09-21T09:50:22.398553+0000","last_became_peered":"2024-09-21T09:50:22.398553+0000","last_unstale":"2024-09-21T09:51:07.206148+0000","last_undegraded":"2024-09-21T09:51:07.206148+0000","last_fullsized":"2024-09-21T09:51:07.206148+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:19.656693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.636193+0000","last_change":"2024-09-21T09:50:22.402415+0000","last_active":"2024-09-21T09:51:07.636193+0000","last_peered":"2024-09-21T09:51:07.636193+0000","last_clean":"2024-09-21T09:51:07.636193+0000","last_became_active":"2024-09-21T09:50:22.401923+0000","last_became_peered":"2024-09-21T09:50:22.401923+0000","last_unstale":"2024-09-21T09:51:07.636193+0000","last_undegraded":"2024-09-21T09:51:07.636193+0000","last_fullsized":"2024-09-21T09:51:07.636193+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:57.139978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.208833+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T09:51:07.208833+0000","last_peered":"2024-09-21T09:51:07.208833+0000","last_clean":"2024-09-21T09:51:07.208833+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T09:51:07.208833+0000","last_undegraded":"2024-09-21T09:51:07.208833+0000","last_fullsized":"2024-09-21T09:51:07.208833+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.636138+0000","last_change":"2024-09-21T09:50:22.401357+0000","last_active":"2024-09-21T09:51:07.636138+0000","last_peered":"2024-09-21T09:51:07.636138+0000","last_clean":"2024-09-21T09:51:07.636138+0000","last_became_active":"2024-09-21T09:50:22.401073+0000","last_became_peered":"2024-09-21T09:50:22.401073+0000","last_unstale":"2024-09-21T09:51:07.636138+0000","last_undegraded":"2024-09-21T09:51:07.636138+0000","last_fullsized":"2024-09-21T09:51:07.636138+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:27.592381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203856+0000","last_change":"2024-09-21T09:50:23.416128+0000","last_active":"2024-09-21T09:51:07.203856+0000","last_peered":"2024-09-21T09:51:07.203856+0000","last_clean":"2024-09-21T09:51:07.203856+0000","last_became_active":"2024-09-21T09:50:23.415802+0000","last_became_peered":"2024-09-21T09:50:23.415802+0000","last_unstale":"2024-09-21T09:51:07.203856+0000","last_undegraded":"2024-09-21T09:51:07.203856+0000","last_fullsized":"2024-09-21T09:51:07.203856+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:14:22.747460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.636085+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T09:51:07.636085+0000","last_peered":"2024-09-21T09:51:07.636085+0000","last_clean":"2024-09-21T09:51:07.636085+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T09:51:07.636085+0000","last_undegraded":"2024-09-21T09:51:07.636085+0000","last_fullsized":"2024-09-21T09:51:07.636085+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205440+0000","last_change":"2024-09-21T09:50:22.930049+0000","last_active":"2024-09-21T09:51:07.205440+0000","last_peered":"2024-09-21T09:51:07.205440+0000","last_clean":"2024-09-21T09:51:07.205440+0000","last_became_active":"2024-09-21T09:50:22.929749+0000","last_became_peered":"2024-09-21T09:50:22.929749+0000","last_unstale":"2024-09-21T09:51:07.205440+0000","last_undegraded":"2024-09-21T09:51:07.205440+0000","last_fullsized":"2024-09-21T09:51:07.205440+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:22:18.074192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209001+0000","last_change":"2024-09-21T09:50:22.391634+0000","last_active":"2024-09-21T09:51:07.209001+0000","last_peered":"2024-09-21T09:51:07.209001+0000","last_clean":"2024-09-21T09:51:07.209001+0000","last_became_active":"2024-09-21T09:50:22.391367+0000","last_became_peered":"2024-09-21T09:50:22.391367+0000","last_unstale":"2024-09-21T09:51:07.209001+0000","last_undegraded":"2024-09-21T09:51:07.209001+0000","last_fullsized":"2024-09-21T09:51:07.209001+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:10.447759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635289+0000","last_change":"2024-09-21T09:50:23.417410+0000","last_active":"2024-09-21T09:51:07.635289+0000","last_peered":"2024-09-21T09:51:07.635289+0000","last_clean":"2024-09-21T09:51:07.635289+0000","last_became_active":"2024-09-21T09:50:23.415908+0000","last_became_peered":"2024-09-21T09:50:23.415908+0000","last_unstale":"2024-09-21T09:51:07.635289+0000","last_undegraded":"2024-09-21T09:51:07.635289+0000","last_fullsized":"2024-09-21T09:51:07.635289+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:06.308139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204885+0000","last_change":"2024-09-21T09:50:22.401877+0000","last_active":"2024-09-21T09:51:07.204885+0000","last_peered":"2024-09-21T09:51:07.204885+0000","last_clean":"2024-09-21T09:51:07.204885+0000","last_became_active":"2024-09-21T09:50:22.401198+0000","last_became_peered":"2024-09-21T09:50:22.401198+0000","last_unstale":"2024-09-21T09:51:07.204885+0000","last_undegraded":"2024-09-21T09:51:07.204885+0000","last_fullsized":"2024-09-21T09:51:07.204885+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:12:50.924087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203638+0000","last_change":"2024-09-21T09:50:23.755321+0000","last_active":"2024-09-21T09:51:07.203638+0000","last_peered":"2024-09-21T09:51:07.203638+0000","last_clean":"2024-09-21T09:51:07.203638+0000","last_became_active":"2024-09-21T09:50:23.754486+0000","last_became_peered":"2024-09-21T09:50:23.754486+0000","last_unstale":"2024-09-21T09:51:07.203638+0000","last_undegraded":"2024-09-21T09:51:07.203638+0000","last_fullsized":"2024-09-21T09:51:07.203638+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:02.322223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205570+0000","last_change":"2024-09-21T09:50:22.399247+0000","last_active":"2024-09-21T09:51:07.205570+0000","last_peered":"2024-09-21T09:51:07.205570+0000","last_clean":"2024-09-21T09:51:07.205570+0000","last_became_active":"2024-09-21T09:50:22.398902+0000","last_became_peered":"2024-09-21T09:50:22.398902+0000","last_unstale":"2024-09-21T09:51:07.205570+0000","last_undegraded":"2024-09-21T09:51:07.205570+0000","last_fullsized":"2024-09-21T09:51:07.205570+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:43.391145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203813+0000","last_change":"2024-09-21T09:50:23.413131+0000","last_active":"2024-09-21T09:51:07.203813+0000","last_peered":"2024-09-21T09:51:07.203813+0000","last_clean":"2024-09-21T09:51:07.203813+0000","last_became_active":"2024-09-21T09:50:23.412860+0000","last_became_peered":"2024-09-21T09:50:23.412860+0000","last_unstale":"2024-09-21T09:51:07.203813+0000","last_undegraded":"2024-09-21T09:51:07.203813+0000","last_fullsized":"2024-09-21T09:51:07.203813+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:53.342708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203810+0000","last_change":"2024-09-21T09:50:22.402098+0000","last_active":"2024-09-21T09:51:07.203810+0000","last_peered":"2024-09-21T09:51:07.203810+0000","last_clean":"2024-09-21T09:51:07.203810+0000","last_became_active":"2024-09-21T09:50:22.401833+0000","last_became_peered":"2024-09-21T09:50:22.401833+0000","last_unstale":"2024-09-21T09:51:07.203810+0000","last_undegraded":"2024-09-21T09:51:07.203810+0000","last_fullsized":"2024-09-21T09:51:07.203810+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:04:00.340198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204100+0000","last_change":"2024-09-21T09:50:23.756279+0000","last_active":"2024-09-21T09:51:07.204100+0000","last_peered":"2024-09-21T09:51:07.204100+0000","last_clean":"2024-09-21T09:51:07.204100+0000","last_became_active":"2024-09-21T09:50:23.755893+0000","last_became_peered":"2024-09-21T09:50:23.755893+0000","last_unstale":"2024-09-21T09:51:07.204100+0000","last_undegraded":"2024-09-21T09:51:07.204100+0000","last_fullsized":"2024-09-21T09:51:07.204100+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:14.663272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204156+0000","last_change":"2024-09-21T09:50:22.400767+0000","last_active":"2024-09-21T09:51:07.204156+0000","last_peered":"2024-09-21T09:51:07.204156+0000","last_clean":"2024-09-21T09:51:07.204156+0000","last_became_active":"2024-09-21T09:50:22.400303+0000","last_became_peered":"2024-09-21T09:50:22.400303+0000","last_unstale":"2024-09-21T09:51:07.204156+0000","last_undegraded":"2024-09-21T09:51:07.204156+0000","last_fullsized":"2024-09-21T09:51:07.204156+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:42.272749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205874+0000","last_change":"2024-09-21T09:50:23.414470+0000","last_active":"2024-09-21T09:51:07.205874+0000","last_peered":"2024-09-21T09:51:07.205874+0000","last_clean":"2024-09-21T09:51:07.205874+0000","last_became_active":"2024-09-21T09:50:23.414126+0000","last_became_peered":"2024-09-21T09:50:23.414126+0000","last_unstale":"2024-09-21T09:51:07.205874+0000","last_undegraded":"2024-09-21T09:51:07.205874+0000","last_fullsized":"2024-09-21T09:51:07.205874+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:28.502144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635885+0000","last_change":"2024-09-21T09:48:37.934335+0000","last_active":"2024-09-21T09:51:07.635885+0000","last_peered":"2024-09-21T09:51:07.635885+0000","last_clean":"2024-09-21T09:51:07.635885+0000","last_became_active":"2024-09-21T09:48:37.933834+0000","last_became_peered":"2024-09-21T09:48:37.933834+0000","last_unstale":"2024-09-21T09:51:07.635885+0000","last_undegraded":"2024-09-21T09:51:07.635885+0000","last_fullsized":"2024-09-21T09:51:07.635885+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:09.157244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":29,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.206164+0000","last_change":"2024-09-21T09:49:14.321731+0000","last_active":"2024-09-21T09:51:08.206164+0000","last_peered":"2024-09-21T09:51:08.206164+0000","last_clean":"2024-09-21T09:51:08.206164+0000","last_became_active":"2024-09-21T09:49:10.174839+0000","last_became_peered":"2024-09-21T09:49:10.174839+0000","last_unstale":"2024-09-21T09:51:08.206164+0000","last_undegraded":"2024-09-21T09:51:08.206164+0000","last_fullsized":"2024-09-21T09:51:08.206164+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:03.713393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010253300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203600+0000","last_change":"2024-09-21T09:50:22.397106+0000","last_active":"2024-09-21T09:51:07.203600+0000","last_peered":"2024-09-21T09:51:07.203600+0000","last_clean":"2024-09-21T09:51:07.203600+0000","last_became_active":"2024-09-21T09:50:22.396789+0000","last_became_peered":"2024-09-21T09:50:22.396789+0000","last_unstale":"2024-09-21T09:51:07.203600+0000","last_undegraded":"2024-09-21T09:51:07.203600+0000","last_fullsized":"2024-09-21T09:51:07.203600+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:17:26.102651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205842+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T09:51:07.205842+0000","last_peered":"2024-09-21T09:51:07.205842+0000","last_clean":"2024-09-21T09:51:07.205842+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T09:51:07.205842+0000","last_undegraded":"2024-09-21T09:51:07.205842+0000","last_fullsized":"2024-09-21T09:51:07.205842+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.583363+0000","last_change":"2024-09-21T09:49:14.324025+0000","last_active":"2024-09-21T09:51:08.583363+0000","last_peered":"2024-09-21T09:51:08.583363+0000","last_clean":"2024-09-21T09:51:08.583363+0000","last_became_active":"2024-09-21T09:49:10.177038+0000","last_became_peered":"2024-09-21T09:49:10.177038+0000","last_unstale":"2024-09-21T09:51:08.583363+0000","last_undegraded":"2024-09-21T09:51:08.583363+0000","last_fullsized":"2024-09-21T09:51:08.583363+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:22.188993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000343004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635278+0000","last_change":"2024-09-21T09:50:22.403544+0000","last_active":"2024-09-21T09:51:07.635278+0000","last_peered":"2024-09-21T09:51:07.635278+0000","last_clean":"2024-09-21T09:51:07.635278+0000","last_became_active":"2024-09-21T09:50:22.403190+0000","last_became_peered":"2024-09-21T09:50:22.403190+0000","last_unstale":"2024-09-21T09:51:07.635278+0000","last_undegraded":"2024-09-21T09:51:07.635278+0000","last_fullsized":"2024-09-21T09:51:07.635278+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:38:42.889542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635475+0000","last_change":"2024-09-21T09:50:23.413161+0000","last_active":"2024-09-21T09:51:07.635475+0000","last_peered":"2024-09-21T09:51:07.635475+0000","last_clean":"2024-09-21T09:51:07.635475+0000","last_became_active":"2024-09-21T09:50:23.412745+0000","last_became_peered":"2024-09-21T09:50:23.412745+0000","last_unstale":"2024-09-21T09:51:07.635475+0000","last_undegraded":"2024-09-21T09:51:07.635475+0000","last_fullsized":"2024-09-21T09:51:07.635475+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:56:38.263798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.205212+0000","last_change":"2024-09-21T09:49:14.317504+0000","last_active":"2024-09-21T09:51:08.205212+0000","last_peered":"2024-09-21T09:51:08.205212+0000","last_clean":"2024-09-21T09:51:08.205212+0000","last_became_active":"2024-09-21T09:49:10.249697+0000","last_became_peered":"2024-09-21T09:49:10.249697+0000","last_unstale":"2024-09-21T09:51:08.205212+0000","last_undegraded":"2024-09-21T09:51:08.205212+0000","last_fullsized":"2024-09-21T09:51:08.205212+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:22.915376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039017899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203858+0000","last_change":"2024-09-21T09:50:22.400809+0000","last_active":"2024-09-21T09:51:07.203858+0000","last_peered":"2024-09-21T09:51:07.203858+0000","last_clean":"2024-09-21T09:51:07.203858+0000","last_became_active":"2024-09-21T09:50:22.400446+0000","last_became_peered":"2024-09-21T09:50:22.400446+0000","last_unstale":"2024-09-21T09:51:07.203858+0000","last_undegraded":"2024-09-21T09:51:07.203858+0000","last_fullsized":"2024-09-21T09:51:07.203858+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:53.001705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205942+0000","last_change":"2024-09-21T09:50:23.754232+0000","last_active":"2024-09-21T09:51:07.205942+0000","last_peered":"2024-09-21T09:51:07.205942+0000","last_clean":"2024-09-21T09:51:07.205942+0000","last_became_active":"2024-09-21T09:50:23.753889+0000","last_became_peered":"2024-09-21T09:50:23.753889+0000","last_unstale":"2024-09-21T09:51:07.205942+0000","last_undegraded":"2024-09-21T09:51:07.205942+0000","last_fullsized":"2024-09-21T09:51:07.205942+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:31.929579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":30,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.206695+0000","last_change":"2024-09-21T09:49:14.321622+0000","last_active":"2024-09-21T09:51:08.206695+0000","last_peered":"2024-09-21T09:51:08.206695+0000","last_clean":"2024-09-21T09:51:08.206695+0000","last_became_active":"2024-09-21T09:49:10.175207+0000","last_became_peered":"2024-09-21T09:49:10.175207+0000","last_unstale":"2024-09-21T09:51:08.206695+0000","last_undegraded":"2024-09-21T09:51:08.206695+0000","last_fullsized":"2024-09-21T09:51:08.206695+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:54.922603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091875599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204167+0000","last_change":"2024-09-21T09:50:22.400816+0000","last_active":"2024-09-21T09:51:07.204167+0000","last_peered":"2024-09-21T09:51:07.204167+0000","last_clean":"2024-09-21T09:51:07.204167+0000","last_became_active":"2024-09-21T09:50:22.400252+0000","last_became_peered":"2024-09-21T09:50:22.400252+0000","last_unstale":"2024-09-21T09:51:07.204167+0000","last_undegraded":"2024-09-21T09:51:07.204167+0000","last_fullsized":"2024-09-21T09:51:07.204167+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:02:09.928786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.206217+0000","last_change":"2024-09-21T09:50:23.414964+0000","last_active":"2024-09-21T09:51:07.206217+0000","last_peered":"2024-09-21T09:51:07.206217+0000","last_clean":"2024-09-21T09:51:07.206217+0000","last_became_active":"2024-09-21T09:50:23.414118+0000","last_became_peered":"2024-09-21T09:50:23.414118+0000","last_unstale":"2024-09-21T09:51:07.206217+0000","last_undegraded":"2024-09-21T09:51:07.206217+0000","last_fullsized":"2024-09-21T09:51:07.206217+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:12.450540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209269+0000","last_change":"2024-09-21T09:50:22.392418+0000","last_active":"2024-09-21T09:51:07.209269+0000","last_peered":"2024-09-21T09:51:07.209269+0000","last_clean":"2024-09-21T09:51:07.209269+0000","last_became_active":"2024-09-21T09:50:22.392156+0000","last_became_peered":"2024-09-21T09:50:22.392156+0000","last_unstale":"2024-09-21T09:51:07.209269+0000","last_undegraded":"2024-09-21T09:51:07.209269+0000","last_fullsized":"2024-09-21T09:51:07.209269+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:57:16.694688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.335613+0000","last_change":"2024-09-21T09:49:14.322235+0000","last_active":"2024-09-21T09:51:08.335613+0000","last_peered":"2024-09-21T09:51:08.335613+0000","last_clean":"2024-09-21T09:51:08.335613+0000","last_became_active":"2024-09-21T09:49:10.176921+0000","last_became_peered":"2024-09-21T09:49:10.176921+0000","last_unstale":"2024-09-21T09:51:08.335613+0000","last_undegraded":"2024-09-21T09:51:08.335613+0000","last_fullsized":"2024-09-21T09:51:08.335613+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:51.116299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034704499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204269+0000","last_change":"2024-09-21T09:50:23.755246+0000","last_active":"2024-09-21T09:51:07.204269+0000","last_peered":"2024-09-21T09:51:07.204269+0000","last_clean":"2024-09-21T09:51:07.204269+0000","last_became_active":"2024-09-21T09:50:23.754824+0000","last_became_peered":"2024-09-21T09:50:23.754824+0000","last_unstale":"2024-09-21T09:51:07.204269+0000","last_undegraded":"2024-09-21T09:51:07.204269+0000","last_fullsized":"2024-09-21T09:51:07.204269+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:33:47.431559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.206243+0000","last_change":"2024-09-21T09:50:22.399629+0000","last_active":"2024-09-21T09:51:07.206243+0000","last_peered":"2024-09-21T09:51:07.206243+0000","last_clean":"2024-09-21T09:51:07.206243+0000","last_became_active":"2024-09-21T09:50:22.399348+0000","last_became_peered":"2024-09-21T09:50:22.399348+0000","last_unstale":"2024-09-21T09:51:07.206243+0000","last_undegraded":"2024-09-21T09:51:07.206243+0000","last_fullsized":"2024-09-21T09:51:07.206243+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:34.623262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.584327+0000","last_change":"2024-09-21T09:49:14.324021+0000","last_active":"2024-09-21T09:51:08.584327+0000","last_peered":"2024-09-21T09:51:08.584327+0000","last_clean":"2024-09-21T09:51:08.584327+0000","last_became_active":"2024-09-21T09:49:10.249765+0000","last_became_peered":"2024-09-21T09:49:10.249765+0000","last_unstale":"2024-09-21T09:51:08.584327+0000","last_undegraded":"2024-09-21T09:51:08.584327+0000","last_fullsized":"2024-09-21T09:51:08.584327+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:13.890155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031296500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635297+0000","last_change":"2024-09-21T09:50:23.755064+0000","last_active":"2024-09-21T09:51:07.635297+0000","last_peered":"2024-09-21T09:51:07.635297+0000","last_clean":"2024-09-21T09:51:07.635297+0000","last_became_active":"2024-09-21T09:50:23.754728+0000","last_became_peered":"2024-09-21T09:50:23.754728+0000","last_unstale":"2024-09-21T09:51:07.635297+0000","last_undegraded":"2024-09-21T09:51:07.635297+0000","last_fullsized":"2024-09-21T09:51:07.635297+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:43:33.571075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203896+0000","last_change":"2024-09-21T09:50:22.400077+0000","last_active":"2024-09-21T09:51:07.203896+0000","last_peered":"2024-09-21T09:51:07.203896+0000","last_clean":"2024-09-21T09:51:07.203896+0000","last_became_active":"2024-09-21T09:50:22.399639+0000","last_became_peered":"2024-09-21T09:50:22.399639+0000","last_unstale":"2024-09-21T09:51:07.203896+0000","last_undegraded":"2024-09-21T09:51:07.203896+0000","last_fullsized":"2024-09-21T09:51:07.203896+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:46.929718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":27,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.206136+0000","last_change":"2024-09-21T09:49:14.322228+0000","last_active":"2024-09-21T09:51:07.206136+0000","last_peered":"2024-09-21T09:51:07.206136+0000","last_clean":"2024-09-21T09:51:07.206136+0000","last_became_active":"2024-09-21T09:49:10.249779+0000","last_became_peered":"2024-09-21T09:49:10.249779+0000","last_unstale":"2024-09-21T09:51:07.206136+0000","last_undegraded":"2024-09-21T09:51:07.206136+0000","last_fullsized":"2024-09-21T09:51:07.206136+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:44.504161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033889199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204001+0000","last_change":"2024-09-21T09:50:22.931457+0000","last_active":"2024-09-21T09:51:07.204001+0000","last_peered":"2024-09-21T09:51:07.204001+0000","last_clean":"2024-09-21T09:51:07.204001+0000","last_became_active":"2024-09-21T09:50:22.931121+0000","last_became_peered":"2024-09-21T09:50:22.931121+0000","last_unstale":"2024-09-21T09:51:07.204001+0000","last_undegraded":"2024-09-21T09:51:07.204001+0000","last_fullsized":"2024-09-21T09:51:07.204001+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:18.905294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635552+0000","last_change":"2024-09-21T09:50:22.399135+0000","last_active":"2024-09-21T09:51:07.635552+0000","last_peered":"2024-09-21T09:51:07.635552+0000","last_clean":"2024-09-21T09:51:07.635552+0000","last_became_active":"2024-09-21T09:50:22.398731+0000","last_became_peered":"2024-09-21T09:50:22.398731+0000","last_unstale":"2024-09-21T09:51:07.635552+0000","last_undegraded":"2024-09-21T09:51:07.635552+0000","last_fullsized":"2024-09-21T09:51:07.635552+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:09.733443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205876+0000","last_change":"2024-09-21T09:49:14.322989+0000","last_active":"2024-09-21T09:51:07.205876+0000","last_peered":"2024-09-21T09:51:07.205876+0000","last_clean":"2024-09-21T09:51:07.205876+0000","last_became_active":"2024-09-21T09:49:10.176476+0000","last_became_peered":"2024-09-21T09:49:10.176476+0000","last_unstale":"2024-09-21T09:51:07.205876+0000","last_undegraded":"2024-09-21T09:51:07.205876+0000","last_fullsized":"2024-09-21T09:51:07.205876+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:03:19.334046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203594+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T09:51:07.203594+0000","last_peered":"2024-09-21T09:51:07.203594+0000","last_clean":"2024-09-21T09:51:07.203594+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T09:51:07.203594+0000","last_undegraded":"2024-09-21T09:51:07.203594+0000","last_fullsized":"2024-09-21T09:51:07.203594+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204006+0000","last_change":"2024-09-21T09:50:23.756473+0000","last_active":"2024-09-21T09:51:07.204006+0000","last_peered":"2024-09-21T09:51:07.204006+0000","last_clean":"2024-09-21T09:51:07.204006+0000","last_became_active":"2024-09-21T09:50:23.755051+0000","last_became_peered":"2024-09-21T09:50:23.755051+0000","last_unstale":"2024-09-21T09:51:07.204006+0000","last_undegraded":"2024-09-21T09:51:07.204006+0000","last_fullsized":"2024-09-21T09:51:07.204006+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:51:14.382136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203766+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T09:51:07.203766+0000","last_peered":"2024-09-21T09:51:07.203766+0000","last_clean":"2024-09-21T09:51:07.203766+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T09:51:07.203766+0000","last_undegraded":"2024-09-21T09:51:07.203766+0000","last_fullsized":"2024-09-21T09:51:07.203766+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.207885+0000","last_change":"2024-09-21T09:51:08.207885+0000","last_active":"2024-09-21T09:51:08.207885+0000","last_peered":"2024-09-21T09:51:08.207885+0000","last_clean":"2024-09-21T09:51:08.207885+0000","last_became_active":"2024-09-21T09:51:08.207526+0000","last_became_peered":"2024-09-21T09:51:08.207526+0000","last_unstale":"2024-09-21T09:51:08.207885+0000","last_undegraded":"2024-09-21T09:51:08.207885+0000","last_fullsized":"2024-09-21T09:51:08.207885+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:31:02.839035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635957+0000","last_change":"2024-09-21T09:50:22.402043+0000","last_active":"2024-09-21T09:51:07.635957+0000","last_peered":"2024-09-21T09:51:07.635957+0000","last_clean":"2024-09-21T09:51:07.635957+0000","last_became_active":"2024-09-21T09:50:22.401686+0000","last_became_peered":"2024-09-21T09:50:22.401686+0000","last_unstale":"2024-09-21T09:51:07.635957+0000","last_undegraded":"2024-09-21T09:51:07.635957+0000","last_fullsized":"2024-09-21T09:51:07.635957+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:28.871212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209109+0000","last_change":"2024-09-21T09:50:22.394664+0000","last_active":"2024-09-21T09:51:07.209109+0000","last_peered":"2024-09-21T09:51:07.209109+0000","last_clean":"2024-09-21T09:51:07.209109+0000","last_became_active":"2024-09-21T09:50:22.394338+0000","last_became_peered":"2024-09-21T09:50:22.394338+0000","last_unstale":"2024-09-21T09:51:07.209109+0000","last_undegraded":"2024-09-21T09:51:07.209109+0000","last_fullsized":"2024-09-21T09:51:07.209109+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:09.429993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203326+0000","last_change":"2024-09-21T09:50:22.929959+0000","last_active":"2024-09-21T09:51:07.203326+0000","last_peered":"2024-09-21T09:51:07.203326+0000","last_clean":"2024-09-21T09:51:07.203326+0000","last_became_active":"2024-09-21T09:50:22.929655+0000","last_became_peered":"2024-09-21T09:50:22.929655+0000","last_unstale":"2024-09-21T09:51:07.203326+0000","last_undegraded":"2024-09-21T09:51:07.203326+0000","last_fullsized":"2024-09-21T09:51:07.203326+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:57.365409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635610+0000","last_change":"2024-09-21T09:50:22.398412+0000","last_active":"2024-09-21T09:51:07.635610+0000","last_peered":"2024-09-21T09:51:07.635610+0000","last_clean":"2024-09-21T09:51:07.635610+0000","last_became_active":"2024-09-21T09:50:22.398108+0000","last_became_peered":"2024-09-21T09:50:22.398108+0000","last_unstale":"2024-09-21T09:51:07.635610+0000","last_undegraded":"2024-09-21T09:51:07.635610+0000","last_fullsized":"2024-09-21T09:51:07.635610+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:11.058999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.206483+0000","last_change":"2024-09-21T09:51:08.206483+0000","last_active":"2024-09-21T09:51:08.206483+0000","last_peered":"2024-09-21T09:51:08.206483+0000","last_clean":"2024-09-21T09:51:08.206483+0000","last_became_active":"2024-09-21T09:51:08.206154+0000","last_became_peered":"2024-09-21T09:51:08.206154+0000","last_unstale":"2024-09-21T09:51:08.206483+0000","last_undegraded":"2024-09-21T09:51:08.206483+0000","last_fullsized":"2024-09-21T09:51:08.206483+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:07:16.789019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.208430+0000","last_change":"2024-09-21T09:51:08.208430+0000","last_active":"2024-09-21T09:51:08.208430+0000","last_peered":"2024-09-21T09:51:08.208430+0000","last_clean":"2024-09-21T09:51:08.208430+0000","last_became_active":"2024-09-21T09:51:08.207843+0000","last_became_peered":"2024-09-21T09:51:08.207843+0000","last_unstale":"2024-09-21T09:51:08.208430+0000","last_undegraded":"2024-09-21T09:51:08.208430+0000","last_fullsized":"2024-09-21T09:51:08.208430+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:38.622103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204085+0000","last_change":"2024-09-21T09:50:22.402812+0000","last_active":"2024-09-21T09:51:07.204085+0000","last_peered":"2024-09-21T09:51:07.204085+0000","last_clean":"2024-09-21T09:51:07.204085+0000","last_became_active":"2024-09-21T09:50:22.402273+0000","last_became_peered":"2024-09-21T09:50:22.402273+0000","last_unstale":"2024-09-21T09:51:07.204085+0000","last_undegraded":"2024-09-21T09:51:07.204085+0000","last_fullsized":"2024-09-21T09:51:07.204085+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:40.454290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635438+0000","last_change":"2024-09-21T09:50:23.417280+0000","last_active":"2024-09-21T09:51:07.635438+0000","last_peered":"2024-09-21T09:51:07.635438+0000","last_clean":"2024-09-21T09:51:07.635438+0000","last_became_active":"2024-09-21T09:50:23.417002+0000","last_became_peered":"2024-09-21T09:50:23.417002+0000","last_unstale":"2024-09-21T09:51:07.635438+0000","last_undegraded":"2024-09-21T09:51:07.635438+0000","last_fullsized":"2024-09-21T09:51:07.635438+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:49.501587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204064+0000","last_change":"2024-09-21T09:50:22.405044+0000","last_active":"2024-09-21T09:51:07.204064+0000","last_peered":"2024-09-21T09:51:07.204064+0000","last_clean":"2024-09-21T09:51:07.204064+0000","last_became_active":"2024-09-21T09:50:22.403898+0000","last_became_peered":"2024-09-21T09:50:22.403898+0000","last_unstale":"2024-09-21T09:51:07.204064+0000","last_undegraded":"2024-09-21T09:51:07.204064+0000","last_fullsized":"2024-09-21T09:51:07.204064+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:23:33.971849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203767+0000","last_change":"2024-09-21T09:50:22.820199+0000","last_active":"2024-09-21T09:51:07.203767+0000","last_peered":"2024-09-21T09:51:07.203767+0000","last_clean":"2024-09-21T09:51:07.203767+0000","last_became_active":"2024-09-21T09:50:22.819651+0000","last_became_peered":"2024-09-21T09:50:22.819651+0000","last_unstale":"2024-09-21T09:51:07.203767+0000","last_undegraded":"2024-09-21T09:51:07.203767+0000","last_fullsized":"2024-09-21T09:51:07.203767+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:16.499801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203664+0000","last_change":"2024-09-21T09:50:22.404585+0000","last_active":"2024-09-21T09:51:07.203664+0000","last_peered":"2024-09-21T09:51:07.203664+0000","last_clean":"2024-09-21T09:51:07.203664+0000","last_became_active":"2024-09-21T09:50:22.404279+0000","last_became_peered":"2024-09-21T09:50:22.404279+0000","last_unstale":"2024-09-21T09:51:07.203664+0000","last_undegraded":"2024-09-21T09:51:07.203664+0000","last_fullsized":"2024-09-21T09:51:07.203664+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:29:22.909031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.207498+0000","last_change":"2024-09-21T09:50:23.415908+0000","last_active":"2024-09-21T09:51:07.207498+0000","last_peered":"2024-09-21T09:51:07.207498+0000","last_clean":"2024-09-21T09:51:07.207498+0000","last_became_active":"2024-09-21T09:50:23.415594+0000","last_became_peered":"2024-09-21T09:50:23.415594+0000","last_unstale":"2024-09-21T09:51:07.207498+0000","last_undegraded":"2024-09-21T09:51:07.207498+0000","last_fullsized":"2024-09-21T09:51:07.207498+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:22.373294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635406+0000","last_change":"2024-09-21T09:50:22.406273+0000","last_active":"2024-09-21T09:51:07.635406+0000","last_peered":"2024-09-21T09:51:07.635406+0000","last_clean":"2024-09-21T09:51:07.635406+0000","last_became_active":"2024-09-21T09:50:22.405984+0000","last_became_peered":"2024-09-21T09:50:22.405984+0000","last_unstale":"2024-09-21T09:51:07.635406+0000","last_undegraded":"2024-09-21T09:51:07.635406+0000","last_fullsized":"2024-09-21T09:51:07.635406+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:39.609068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203973+0000","last_change":"2024-09-21T09:50:23.414339+0000","last_active":"2024-09-21T09:51:07.203973+0000","last_peered":"2024-09-21T09:51:07.203973+0000","last_clean":"2024-09-21T09:51:07.203973+0000","last_became_active":"2024-09-21T09:50:23.414046+0000","last_became_peered":"2024-09-21T09:50:23.414046+0000","last_unstale":"2024-09-21T09:51:07.203973+0000","last_undegraded":"2024-09-21T09:51:07.203973+0000","last_fullsized":"2024-09-21T09:51:07.203973+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:14.632255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209185+0000","last_change":"2024-09-21T09:50:22.397124+0000","last_active":"2024-09-21T09:51:07.209185+0000","last_peered":"2024-09-21T09:51:07.209185+0000","last_clean":"2024-09-21T09:51:07.209185+0000","last_became_active":"2024-09-21T09:50:22.396873+0000","last_became_peered":"2024-09-21T09:50:22.396873+0000","last_unstale":"2024-09-21T09:51:07.209185+0000","last_undegraded":"2024-09-21T09:51:07.209185+0000","last_fullsized":"2024-09-21T09:51:07.209185+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:46:54.625943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203686+0000","last_change":"2024-09-21T09:50:23.416269+0000","last_active":"2024-09-21T09:51:07.203686+0000","last_peered":"2024-09-21T09:51:07.203686+0000","last_clean":"2024-09-21T09:51:07.203686+0000","last_became_active":"2024-09-21T09:50:23.415890+0000","last_became_peered":"2024-09-21T09:50:23.415890+0000","last_unstale":"2024-09-21T09:51:07.203686+0000","last_undegraded":"2024-09-21T09:51:07.203686+0000","last_fullsized":"2024-09-21T09:51:07.203686+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:39.754711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209251+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T09:51:07.209251+0000","last_peered":"2024-09-21T09:51:07.209251+0000","last_clean":"2024-09-21T09:51:07.209251+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T09:51:07.209251+0000","last_undegraded":"2024-09-21T09:51:07.209251+0000","last_fullsized":"2024-09-21T09:51:07.209251+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203418+0000","last_change":"2024-09-21T09:50:22.929558+0000","last_active":"2024-09-21T09:51:07.203418+0000","last_peered":"2024-09-21T09:51:07.203418+0000","last_clean":"2024-09-21T09:51:07.203418+0000","last_became_active":"2024-09-21T09:50:22.929186+0000","last_became_peered":"2024-09-21T09:50:22.929186+0000","last_unstale":"2024-09-21T09:51:07.203418+0000","last_undegraded":"2024-09-21T09:51:07.203418+0000","last_fullsized":"2024-09-21T09:51:07.203418+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:15:32.908972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205126+0000","last_change":"2024-09-21T09:50:22.394608+0000","last_active":"2024-09-21T09:51:07.205126+0000","last_peered":"2024-09-21T09:51:07.205126+0000","last_clean":"2024-09-21T09:51:07.205126+0000","last_became_active":"2024-09-21T09:50:22.394260+0000","last_became_peered":"2024-09-21T09:50:22.394260+0000","last_unstale":"2024-09-21T09:51:07.205126+0000","last_undegraded":"2024-09-21T09:51:07.205126+0000","last_fullsized":"2024-09-21T09:51:07.205126+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:34:00.389844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203407+0000","last_change":"2024-09-21T09:50:23.756088+0000","last_active":"2024-09-21T09:51:07.203407+0000","last_peered":"2024-09-21T09:51:07.203407+0000","last_clean":"2024-09-21T09:51:07.203407+0000","last_became_active":"2024-09-21T09:50:23.755515+0000","last_became_peered":"2024-09-21T09:50:23.755515+0000","last_unstale":"2024-09-21T09:51:07.203407+0000","last_undegraded":"2024-09-21T09:51:07.203407+0000","last_fullsized":"2024-09-21T09:51:07.203407+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:35.236821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635565+0000","last_change":"2024-09-21T09:50:22.401825+0000","last_active":"2024-09-21T09:51:07.635565+0000","last_peered":"2024-09-21T09:51:07.635565+0000","last_clean":"2024-09-21T09:51:07.635565+0000","last_became_active":"2024-09-21T09:50:22.401550+0000","last_became_peered":"2024-09-21T09:50:22.401550+0000","last_unstale":"2024-09-21T09:51:07.635565+0000","last_undegraded":"2024-09-21T09:51:07.635565+0000","last_fullsized":"2024-09-21T09:51:07.635565+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:08.792265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234019,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28368,"kb_used_data":1344,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716784,"statfs":{"total":95995035648,"available":95965986816,"internally_reserved":0,"allocated":1376256,"data_stored":925421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.36299999999999999,"5min":0.312,"15min":0.312},"max":{"1min":0.71499999999999997,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.40000000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.60399999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.41299999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.58099999999999996,"15min":0.58899999999999997},"min":{"1min":0.34499999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.755,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.755,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59399999999999997,"15min":0.59499999999999997},"min":{"1min":0.436,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76900000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.48199999999999998,"15min":0.48299999999999998},"min":{"1min":0.32900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.69899999999999995,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.52300000000000002,"15min":0.52200000000000002},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.73199999999999998,"5min":0.871,"15min":0.871},"last":0.71099999999999997}]},{"osd":5,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.81000000000000005,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.54400000000000004,"15min":0.53800000000000003},"min":{"1min":0.38300000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.72499999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.57399999999999995,"15min":0.56999999999999995},"min":{"1min":0.38200000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79400000000000004,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40999999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77000000000000002,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.68400000000000005}]}]},{"osd":6,"up_from":46,"seq":197568495657,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28364,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716788,"statfs":{"total":95995035648,"available":95965990912,"internally_reserved":0,"allocated":1372160,"data_stored":925625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51500000000000001,"15min":0.50700000000000001},"min":{"1min":0.373,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.53000000000000003,"15min":0.51200000000000001},"min":{"1min":0.40000000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76900000000000002,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.76900000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.59899999999999998,"15min":0.54700000000000004},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78100000000000003,"5min":6.008,"15min":6.008},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54000000000000004,"15min":0.53100000000000003},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.84299999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59299999999999997,"15min":0.53700000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.80700000000000005,"5min":6.0309999999999997,"15min":6.0309999999999997},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.40500000000000003,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.78900000000000003}]},{"osd":3,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.65800000000000003,"15min":0.58199999999999996},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.481,"5min":10.481,"15min":10.481},"last":0.88},{"interface":"front","average":{"1min":1.056,"5min":0.72299999999999998,"15min":0.60699999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.518000000000001,"5min":10.518000000000001,"15min":10.518000000000001},"last":0.82799999999999996}]},{"osd":4,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.437,"15min":0.42899999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.622,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.44700000000000001,"15min":0.42899999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.69299999999999995,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.69299999999999995}]},{"osd":5,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.45200000000000001,"15min":0.44},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.47999999999999998,"15min":0.46300000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.81200000000000006}]},{"osd":7,"last update":"Sat Sep 21 09:50:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.54600000000000004,"15min":0.53900000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.50700000000000001,"15min":0.505},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.86299999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757294,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":770048,"data_stored":334275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.57399999999999995,"15min":0.56100000000000005},"min":{"1min":0.34100000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.536,"5min":2.536,"15min":2.536},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.54400000000000004,"15min":0.55300000000000005},"min":{"1min":0.35499999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.73899999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.68200000000000005}]},{"osd":1,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.72199999999999998,"15min":0.79100000000000004},"min":{"1min":0.38,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71999999999999997,"5min":7.2190000000000003,"15min":7.2190000000000003},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.77100000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.48399999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.70099999999999996,"15min":0.77500000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.82199999999999995,"5min":7.1619999999999999,"15min":7.1619999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59199999999999997,"15min":0.60199999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.75800000000000001,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.55600000000000005,"15min":0.56399999999999995},"min":{"1min":0.376,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.67800000000000005,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.499},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.753,"15min":0.82499999999999996},"min":{"1min":0.42999999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78300000000000003,"5min":7.181,"15min":7.181},"last":0.60199999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.437,"15min":0.433},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.60099999999999998,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.46600000000000003,"15min":0.46100000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.63,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.434}]},{"osd":6,"last update":"Sat Sep 21 09:51:16 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.318,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.70599999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.753,"5min":1.179,"15min":1.179},"last":0.38}]},{"osd":7,"last update":"Sat Sep 21 09:50:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.59599999999999997,"15min":0.61399999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70199999999999996,"5min":1.24,"15min":1.24},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.54200000000000004,"15min":0.55100000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68500000000000005,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.40899999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018930,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31880,"kb_used_data":764,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713272,"statfs":{"total":95995035648,"available":95962390528,"internally_reserved":0,"allocated":782336,"data_stored":335314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.72699999999999998,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.70899999999999996,"15min":0.77900000000000003},"min":{"1min":0.39400000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.71699999999999997,"5min":5.4029999999999996,"15min":5.4029999999999996},"last":0.57599999999999996}]},{"osd":1,"last update":"Sat Sep 21 09:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.38900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.80800000000000005,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.495,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.39700000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.67400000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.53100000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52100000000000002,"15min":0.52400000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.60999999999999999,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.58299999999999996,"15min":0.58399999999999996},"min":{"1min":0.41399999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.72099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.66000000000000003}]},{"osd":3,"last update":"Sat Sep 21 09:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.80500000000000005,"15min":0.80900000000000005},"min":{"1min":0.45200000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.747,"5min":8.0060000000000002,"15min":8.0060000000000002},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.72799999999999998,"15min":0.78900000000000003},"min":{"1min":0.45900000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80100000000000005,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.621}]},{"osd":5,"last update":"Sat Sep 21 09:50:50 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.41099999999999998,"15min":0.38800000000000001},"min":{"1min":0.311,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67400000000000004,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.437},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.435,"15min":0.40200000000000002},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69699999999999995,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:50:14 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.53200000000000003,"15min":0.52500000000000002},"min":{"1min":0.432,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75900000000000001,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.52300000000000002,"15min":0.51500000000000001},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.75900000000000001,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:50:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56200000000000006,"15min":0.56699999999999995},"min":{"1min":0.374,"5min":0.316,"15min":0.316},"max":{"1min":0.67400000000000004,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.58699999999999997,"15min":0.59499999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.74199999999999999,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.71899999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182455,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":333694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:51:09 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.38800000000000001,"15min":0.39300000000000002},"min":{"1min":0.215,"5min":0.185,"15min":0.185},"max":{"1min":0.65900000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.38},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.40699999999999997,"15min":0.432},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.60399999999999998,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:51:09 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.42499999999999999,"15min":0.44900000000000001},"min":{"1min":0.317,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.48699999999999999,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.432},{"interface":"front","average":{"1min":0.34399999999999997,"5min":0.39100000000000001,"15min":0.40699999999999997},"min":{"1min":0.23599999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.58999999999999997,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.41099999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:51:09 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.437,"15min":0.45600000000000002},"min":{"1min":0.29799999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.48399999999999999,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.42399999999999999,"15min":0.45200000000000001},"min":{"1min":0.26000000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.46300000000000002,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.47699999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:50:31 2024","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":0.746,"15min":0.65300000000000002},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":15.769,"5min":15.769,"15min":15.769},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.433,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81499999999999995,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Sep 21 09:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.59799999999999998,"15min":0.61399999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.67900000000000005,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.59099999999999997,"15min":0.60299999999999998},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.37},"max":{"1min":0.76800000000000002,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.76100000000000001,"15min":0.66600000000000004},"min":{"1min":0.41799999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.72299999999999998,"5min":15.784000000000001,"15min":15.784000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.59799999999999998,"15min":0.60399999999999998},"min":{"1min":0.36699999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.83699999999999997,"5min":0.877,"15min":0.877},"last":0.60299999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:50:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79300000000000004,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.60299999999999998,"15min":0.60799999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.73999999999999999,"5min":0.755,"15min":0.755},"last":0.65400000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31868,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713284,"statfs":{"total":95995035648,"available":95962402816,"internally_reserved":0,"allocated":765952,"data_stored":334220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:44 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.35699999999999998,"15min":0.35299999999999998},"min":{"1min":0.222,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.51900000000000002,"5min":0.879,"15min":0.879},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.37,"15min":0.36399999999999999},"min":{"1min":0.251,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.51000000000000001,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.27700000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:50:44 2024","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.36799999999999999,"15min":0.36499999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.47799999999999998,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.317},{"interface":"front","average":{"1min":0.372,"5min":0.38500000000000001,"15min":0.38100000000000001},"min":{"1min":0.28100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":0.50700000000000001,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:51:10 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.47799999999999998,"15min":0.48399999999999999},"min":{"1min":0.33500000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.147,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.443,"15min":0.44700000000000001},"min":{"1min":0.318,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.57399999999999995,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:50:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.59399999999999997,"15min":0.59699999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.59599999999999997,"15min":0.59699999999999998},"min":{"1min":0.46100000000000002,"5min":0.373,"15min":0.373},"max":{"1min":0.74399999999999999,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.56699999999999995}]},{"osd":5,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.57699999999999996,"15min":0.58499999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76300000000000001,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.57599999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56399999999999995,"15min":0.56299999999999994},"min":{"1min":0.41899999999999998,"5min":0.372,"15min":0.372},"max":{"1min":0.72899999999999998,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.622,"15min":0.62},"min":{"1min":0.48899999999999999,"5min":0.376,"15min":0.376},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.72599999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:50:38 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.629,"15min":0.63800000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.75700000000000001,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.61299999999999999,"15min":0.61499999999999999},"min":{"1min":0.42799999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.746}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27760,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717392,"statfs":{"total":95995035648,"available":95966609408,"internally_reserved":0,"allocated":757760,"data_stored":333350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:19 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.33400000000000002,"15min":0.317},"min":{"1min":0.23300000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.55000000000000004,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.34799999999999998,"15min":0.32800000000000001},"min":{"1min":0.23699999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.50900000000000001,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42799999999999999,"15min":0.41099999999999998},"min":{"1min":0.33500000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.54600000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.39000000000000001,"15min":0.36699999999999999},"min":{"1min":0.255,"5min":0.222,"15min":0.222},"max":{"1min":0.45600000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.62}]},{"osd":3,"last update":"Sat Sep 21 09:51:10 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.39900000000000002,"15min":0.39100000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.48099999999999998,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.42999999999999999,"15min":0.42599999999999999},"min":{"1min":0.31900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.51100000000000001,"5min":0.63,"15min":0.63},"last":0.66900000000000004}]},{"osd":4,"last update":"Sat Sep 21 09:50:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.35999999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.80900000000000005,"5min":0.88,"15min":0.88},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56799999999999995,"15min":0.56599999999999995},"min":{"1min":0.39500000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.76200000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57699999999999996,"15min":0.58299999999999996},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.45700000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.73499999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:51:14 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.60899999999999999,"15min":0.61399999999999999},"min":{"1min":0.44500000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.79300000000000004,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.60699999999999998,"15min":0.60999999999999999},"min":{"1min":0.40600000000000003,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.86599999999999999,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.69799999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:50:36 2024","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":0.68799999999999994,"15min":0.61499999999999999},"min":{"1min":0.46600000000000003,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":10.279,"5min":10.279,"15min":10.279},"last":0.747},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59899999999999998,"15min":0.59799999999999998},"min":{"1min":0.38,"5min":0.378,"15min":0.378},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.81499999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32448,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712704,"statfs":{"total":95995035648,"available":95961808896,"internally_reserved":0,"allocated":1359872,"data_stored":924576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.34499999999999997,"15min":0.33500000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":0.54700000000000004,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.315},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.35099999999999998,"15min":0.33500000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.55100000000000005,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.35799999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:50:46 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.38900000000000001,"15min":0.38400000000000001},"min":{"1min":0.252,"5min":0.246,"15min":0.246},"max":{"1min":0.54800000000000004,"5min":0.622,"15min":0.622},"last":0.378},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.432,"15min":0.433},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60799999999999998,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.42899999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:51:08 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.42099999999999999,"15min":0.435},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.53200000000000003,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.38,"5min":0.436,"15min":0.46500000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.65500000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.58199999999999996,"15min":0.60099999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.88,"5min":1.036,"15min":1.036},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.62,"15min":0.61599999999999999},"min":{"1min":0.40300000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.80600000000000005,"5min":4.4210000000000003,"15min":4.4210000000000003},"last":0.55500000000000005}]},{"osd":5,"last update":"Sat Sep 21 09:50:51 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.61799999999999999,"15min":0.60799999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":4.8019999999999996,"5min":4.8019999999999996,"15min":4.8019999999999996},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.60599999999999998,"15min":0.63300000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81599999999999995,"5min":1.254,"15min":1.254},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:51:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.57999999999999996,"15min":0.58999999999999997},"min":{"1min":0.40999999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70599999999999996,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.71599999999999997,"15min":0.65700000000000003},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.80400000000000005,"5min":13.663,"15min":13.663},"last":0.746}]},{"osd":7,"last update":"Sat Sep 21 09:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.61099999999999999,"15min":0.62},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89400000000000002,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.64300000000000002,"15min":0.65300000000000002},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90800000000000003,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.77400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:51:21.598 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:51:21.803 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:22.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:22 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:22 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:22 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:23.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:23 smithi032 ceph-mon[40061]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:23 smithi032 ceph-mon[29191]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:23.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:23 smithi045 ceph-mon[33087]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:23.576 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:51:23.577 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T09:51:23.934 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:51:23] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2024-09-21T09:51:24.193 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":85,"stamp":"2024-09-21T09:51:23.165243+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":236232,"kb_used_data":7760,"kb_used_omap":12,"kb_used_meta":228339,"kb_avail":749724984,"statfs":{"total":767960285184,"available":767718383616,"internally_reserved":0,"allocated":7946240,"data_stored":4446475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":233819704},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002858"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204424+0000","last_change":"2024-09-21T09:50:22.930135+0000","last_active":"2024-09-21T09:51:07.204424+0000","last_peered":"2024-09-21T09:51:07.204424+0000","last_clean":"2024-09-21T09:51:07.204424+0000","last_became_active":"2024-09-21T09:50:22.929856+0000","last_became_peered":"2024-09-21T09:50:22.929856+0000","last_unstale":"2024-09-21T09:51:07.204424+0000","last_undegraded":"2024-09-21T09:51:07.204424+0000","last_fullsized":"2024-09-21T09:51:07.204424+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:02.058108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635768+0000","last_change":"2024-09-21T09:50:22.400322+0000","last_active":"2024-09-21T09:51:07.635768+0000","last_peered":"2024-09-21T09:51:07.635768+0000","last_clean":"2024-09-21T09:51:07.635768+0000","last_became_active":"2024-09-21T09:50:22.400017+0000","last_became_peered":"2024-09-21T09:50:22.400017+0000","last_unstale":"2024-09-21T09:51:07.635768+0000","last_undegraded":"2024-09-21T09:51:07.635768+0000","last_fullsized":"2024-09-21T09:51:07.635768+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:24:27.572098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209060+0000","last_change":"2024-09-21T09:50:22.931558+0000","last_active":"2024-09-21T09:51:07.209060+0000","last_peered":"2024-09-21T09:51:07.209060+0000","last_clean":"2024-09-21T09:51:07.209060+0000","last_became_active":"2024-09-21T09:50:22.931278+0000","last_became_peered":"2024-09-21T09:50:22.931278+0000","last_unstale":"2024-09-21T09:51:07.209060+0000","last_undegraded":"2024-09-21T09:51:07.209060+0000","last_fullsized":"2024-09-21T09:51:07.209060+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:12.076005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635285+0000","last_change":"2024-09-21T09:50:22.407053+0000","last_active":"2024-09-21T09:51:07.635285+0000","last_peered":"2024-09-21T09:51:07.635285+0000","last_clean":"2024-09-21T09:51:07.635285+0000","last_became_active":"2024-09-21T09:50:22.406593+0000","last_became_peered":"2024-09-21T09:50:22.406593+0000","last_unstale":"2024-09-21T09:51:07.635285+0000","last_undegraded":"2024-09-21T09:51:07.635285+0000","last_fullsized":"2024-09-21T09:51:07.635285+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:50.832120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635514+0000","last_change":"2024-09-21T09:50:23.416113+0000","last_active":"2024-09-21T09:51:07.635514+0000","last_peered":"2024-09-21T09:51:07.635514+0000","last_clean":"2024-09-21T09:51:07.635514+0000","last_became_active":"2024-09-21T09:50:23.415818+0000","last_became_peered":"2024-09-21T09:50:23.415818+0000","last_unstale":"2024-09-21T09:51:07.635514+0000","last_undegraded":"2024-09-21T09:51:07.635514+0000","last_fullsized":"2024-09-21T09:51:07.635514+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:26.685970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635721+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T09:51:07.635721+0000","last_peered":"2024-09-21T09:51:07.635721+0000","last_clean":"2024-09-21T09:51:07.635721+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T09:51:07.635721+0000","last_undegraded":"2024-09-21T09:51:07.635721+0000","last_fullsized":"2024-09-21T09:51:07.635721+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203808+0000","last_change":"2024-09-21T09:50:22.930261+0000","last_active":"2024-09-21T09:51:07.203808+0000","last_peered":"2024-09-21T09:51:07.203808+0000","last_clean":"2024-09-21T09:51:07.203808+0000","last_became_active":"2024-09-21T09:50:22.929920+0000","last_became_peered":"2024-09-21T09:50:22.929920+0000","last_unstale":"2024-09-21T09:51:07.203808+0000","last_undegraded":"2024-09-21T09:51:07.203808+0000","last_fullsized":"2024-09-21T09:51:07.203808+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:50.274508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.206148+0000","last_change":"2024-09-21T09:50:22.398833+0000","last_active":"2024-09-21T09:51:07.206148+0000","last_peered":"2024-09-21T09:51:07.206148+0000","last_clean":"2024-09-21T09:51:07.206148+0000","last_became_active":"2024-09-21T09:50:22.398553+0000","last_became_peered":"2024-09-21T09:50:22.398553+0000","last_unstale":"2024-09-21T09:51:07.206148+0000","last_undegraded":"2024-09-21T09:51:07.206148+0000","last_fullsized":"2024-09-21T09:51:07.206148+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:19.656693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.636193+0000","last_change":"2024-09-21T09:50:22.402415+0000","last_active":"2024-09-21T09:51:07.636193+0000","last_peered":"2024-09-21T09:51:07.636193+0000","last_clean":"2024-09-21T09:51:07.636193+0000","last_became_active":"2024-09-21T09:50:22.401923+0000","last_became_peered":"2024-09-21T09:50:22.401923+0000","last_unstale":"2024-09-21T09:51:07.636193+0000","last_undegraded":"2024-09-21T09:51:07.636193+0000","last_fullsized":"2024-09-21T09:51:07.636193+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:57.139978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.208833+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T09:51:07.208833+0000","last_peered":"2024-09-21T09:51:07.208833+0000","last_clean":"2024-09-21T09:51:07.208833+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T09:51:07.208833+0000","last_undegraded":"2024-09-21T09:51:07.208833+0000","last_fullsized":"2024-09-21T09:51:07.208833+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.636138+0000","last_change":"2024-09-21T09:50:22.401357+0000","last_active":"2024-09-21T09:51:07.636138+0000","last_peered":"2024-09-21T09:51:07.636138+0000","last_clean":"2024-09-21T09:51:07.636138+0000","last_became_active":"2024-09-21T09:50:22.401073+0000","last_became_peered":"2024-09-21T09:50:22.401073+0000","last_unstale":"2024-09-21T09:51:07.636138+0000","last_undegraded":"2024-09-21T09:51:07.636138+0000","last_fullsized":"2024-09-21T09:51:07.636138+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:27.592381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203856+0000","last_change":"2024-09-21T09:50:23.416128+0000","last_active":"2024-09-21T09:51:07.203856+0000","last_peered":"2024-09-21T09:51:07.203856+0000","last_clean":"2024-09-21T09:51:07.203856+0000","last_became_active":"2024-09-21T09:50:23.415802+0000","last_became_peered":"2024-09-21T09:50:23.415802+0000","last_unstale":"2024-09-21T09:51:07.203856+0000","last_undegraded":"2024-09-21T09:51:07.203856+0000","last_fullsized":"2024-09-21T09:51:07.203856+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:14:22.747460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.636085+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T09:51:07.636085+0000","last_peered":"2024-09-21T09:51:07.636085+0000","last_clean":"2024-09-21T09:51:07.636085+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T09:51:07.636085+0000","last_undegraded":"2024-09-21T09:51:07.636085+0000","last_fullsized":"2024-09-21T09:51:07.636085+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205440+0000","last_change":"2024-09-21T09:50:22.930049+0000","last_active":"2024-09-21T09:51:07.205440+0000","last_peered":"2024-09-21T09:51:07.205440+0000","last_clean":"2024-09-21T09:51:07.205440+0000","last_became_active":"2024-09-21T09:50:22.929749+0000","last_became_peered":"2024-09-21T09:50:22.929749+0000","last_unstale":"2024-09-21T09:51:07.205440+0000","last_undegraded":"2024-09-21T09:51:07.205440+0000","last_fullsized":"2024-09-21T09:51:07.205440+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:22:18.074192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209001+0000","last_change":"2024-09-21T09:50:22.391634+0000","last_active":"2024-09-21T09:51:07.209001+0000","last_peered":"2024-09-21T09:51:07.209001+0000","last_clean":"2024-09-21T09:51:07.209001+0000","last_became_active":"2024-09-21T09:50:22.391367+0000","last_became_peered":"2024-09-21T09:50:22.391367+0000","last_unstale":"2024-09-21T09:51:07.209001+0000","last_undegraded":"2024-09-21T09:51:07.209001+0000","last_fullsized":"2024-09-21T09:51:07.209001+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:10.447759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635289+0000","last_change":"2024-09-21T09:50:23.417410+0000","last_active":"2024-09-21T09:51:07.635289+0000","last_peered":"2024-09-21T09:51:07.635289+0000","last_clean":"2024-09-21T09:51:07.635289+0000","last_became_active":"2024-09-21T09:50:23.415908+0000","last_became_peered":"2024-09-21T09:50:23.415908+0000","last_unstale":"2024-09-21T09:51:07.635289+0000","last_undegraded":"2024-09-21T09:51:07.635289+0000","last_fullsized":"2024-09-21T09:51:07.635289+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:06.308139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204885+0000","last_change":"2024-09-21T09:50:22.401877+0000","last_active":"2024-09-21T09:51:07.204885+0000","last_peered":"2024-09-21T09:51:07.204885+0000","last_clean":"2024-09-21T09:51:07.204885+0000","last_became_active":"2024-09-21T09:50:22.401198+0000","last_became_peered":"2024-09-21T09:50:22.401198+0000","last_unstale":"2024-09-21T09:51:07.204885+0000","last_undegraded":"2024-09-21T09:51:07.204885+0000","last_fullsized":"2024-09-21T09:51:07.204885+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:12:50.924087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203638+0000","last_change":"2024-09-21T09:50:23.755321+0000","last_active":"2024-09-21T09:51:07.203638+0000","last_peered":"2024-09-21T09:51:07.203638+0000","last_clean":"2024-09-21T09:51:07.203638+0000","last_became_active":"2024-09-21T09:50:23.754486+0000","last_became_peered":"2024-09-21T09:50:23.754486+0000","last_unstale":"2024-09-21T09:51:07.203638+0000","last_undegraded":"2024-09-21T09:51:07.203638+0000","last_fullsized":"2024-09-21T09:51:07.203638+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:02.322223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205570+0000","last_change":"2024-09-21T09:50:22.399247+0000","last_active":"2024-09-21T09:51:07.205570+0000","last_peered":"2024-09-21T09:51:07.205570+0000","last_clean":"2024-09-21T09:51:07.205570+0000","last_became_active":"2024-09-21T09:50:22.398902+0000","last_became_peered":"2024-09-21T09:50:22.398902+0000","last_unstale":"2024-09-21T09:51:07.205570+0000","last_undegraded":"2024-09-21T09:51:07.205570+0000","last_fullsized":"2024-09-21T09:51:07.205570+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:43.391145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203813+0000","last_change":"2024-09-21T09:50:23.413131+0000","last_active":"2024-09-21T09:51:07.203813+0000","last_peered":"2024-09-21T09:51:07.203813+0000","last_clean":"2024-09-21T09:51:07.203813+0000","last_became_active":"2024-09-21T09:50:23.412860+0000","last_became_peered":"2024-09-21T09:50:23.412860+0000","last_unstale":"2024-09-21T09:51:07.203813+0000","last_undegraded":"2024-09-21T09:51:07.203813+0000","last_fullsized":"2024-09-21T09:51:07.203813+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:53.342708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203810+0000","last_change":"2024-09-21T09:50:22.402098+0000","last_active":"2024-09-21T09:51:07.203810+0000","last_peered":"2024-09-21T09:51:07.203810+0000","last_clean":"2024-09-21T09:51:07.203810+0000","last_became_active":"2024-09-21T09:50:22.401833+0000","last_became_peered":"2024-09-21T09:50:22.401833+0000","last_unstale":"2024-09-21T09:51:07.203810+0000","last_undegraded":"2024-09-21T09:51:07.203810+0000","last_fullsized":"2024-09-21T09:51:07.203810+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:04:00.340198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204100+0000","last_change":"2024-09-21T09:50:23.756279+0000","last_active":"2024-09-21T09:51:07.204100+0000","last_peered":"2024-09-21T09:51:07.204100+0000","last_clean":"2024-09-21T09:51:07.204100+0000","last_became_active":"2024-09-21T09:50:23.755893+0000","last_became_peered":"2024-09-21T09:50:23.755893+0000","last_unstale":"2024-09-21T09:51:07.204100+0000","last_undegraded":"2024-09-21T09:51:07.204100+0000","last_fullsized":"2024-09-21T09:51:07.204100+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:14.663272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204156+0000","last_change":"2024-09-21T09:50:22.400767+0000","last_active":"2024-09-21T09:51:07.204156+0000","last_peered":"2024-09-21T09:51:07.204156+0000","last_clean":"2024-09-21T09:51:07.204156+0000","last_became_active":"2024-09-21T09:50:22.400303+0000","last_became_peered":"2024-09-21T09:50:22.400303+0000","last_unstale":"2024-09-21T09:51:07.204156+0000","last_undegraded":"2024-09-21T09:51:07.204156+0000","last_fullsized":"2024-09-21T09:51:07.204156+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:42.272749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205874+0000","last_change":"2024-09-21T09:50:23.414470+0000","last_active":"2024-09-21T09:51:07.205874+0000","last_peered":"2024-09-21T09:51:07.205874+0000","last_clean":"2024-09-21T09:51:07.205874+0000","last_became_active":"2024-09-21T09:50:23.414126+0000","last_became_peered":"2024-09-21T09:50:23.414126+0000","last_unstale":"2024-09-21T09:51:07.205874+0000","last_undegraded":"2024-09-21T09:51:07.205874+0000","last_fullsized":"2024-09-21T09:51:07.205874+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:28.502144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635885+0000","last_change":"2024-09-21T09:48:37.934335+0000","last_active":"2024-09-21T09:51:07.635885+0000","last_peered":"2024-09-21T09:51:07.635885+0000","last_clean":"2024-09-21T09:51:07.635885+0000","last_became_active":"2024-09-21T09:48:37.933834+0000","last_became_peered":"2024-09-21T09:48:37.933834+0000","last_unstale":"2024-09-21T09:51:07.635885+0000","last_undegraded":"2024-09-21T09:51:07.635885+0000","last_fullsized":"2024-09-21T09:51:07.635885+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:09.157244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":29,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.206164+0000","last_change":"2024-09-21T09:49:14.321731+0000","last_active":"2024-09-21T09:51:08.206164+0000","last_peered":"2024-09-21T09:51:08.206164+0000","last_clean":"2024-09-21T09:51:08.206164+0000","last_became_active":"2024-09-21T09:49:10.174839+0000","last_became_peered":"2024-09-21T09:49:10.174839+0000","last_unstale":"2024-09-21T09:51:08.206164+0000","last_undegraded":"2024-09-21T09:51:08.206164+0000","last_fullsized":"2024-09-21T09:51:08.206164+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:03.713393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010253300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203600+0000","last_change":"2024-09-21T09:50:22.397106+0000","last_active":"2024-09-21T09:51:07.203600+0000","last_peered":"2024-09-21T09:51:07.203600+0000","last_clean":"2024-09-21T09:51:07.203600+0000","last_became_active":"2024-09-21T09:50:22.396789+0000","last_became_peered":"2024-09-21T09:50:22.396789+0000","last_unstale":"2024-09-21T09:51:07.203600+0000","last_undegraded":"2024-09-21T09:51:07.203600+0000","last_fullsized":"2024-09-21T09:51:07.203600+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:17:26.102651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205842+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T09:51:07.205842+0000","last_peered":"2024-09-21T09:51:07.205842+0000","last_clean":"2024-09-21T09:51:07.205842+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T09:51:07.205842+0000","last_undegraded":"2024-09-21T09:51:07.205842+0000","last_fullsized":"2024-09-21T09:51:07.205842+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.583363+0000","last_change":"2024-09-21T09:49:14.324025+0000","last_active":"2024-09-21T09:51:08.583363+0000","last_peered":"2024-09-21T09:51:08.583363+0000","last_clean":"2024-09-21T09:51:08.583363+0000","last_became_active":"2024-09-21T09:49:10.177038+0000","last_became_peered":"2024-09-21T09:49:10.177038+0000","last_unstale":"2024-09-21T09:51:08.583363+0000","last_undegraded":"2024-09-21T09:51:08.583363+0000","last_fullsized":"2024-09-21T09:51:08.583363+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:22.188993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000343004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635278+0000","last_change":"2024-09-21T09:50:22.403544+0000","last_active":"2024-09-21T09:51:07.635278+0000","last_peered":"2024-09-21T09:51:07.635278+0000","last_clean":"2024-09-21T09:51:07.635278+0000","last_became_active":"2024-09-21T09:50:22.403190+0000","last_became_peered":"2024-09-21T09:50:22.403190+0000","last_unstale":"2024-09-21T09:51:07.635278+0000","last_undegraded":"2024-09-21T09:51:07.635278+0000","last_fullsized":"2024-09-21T09:51:07.635278+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:38:42.889542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635475+0000","last_change":"2024-09-21T09:50:23.413161+0000","last_active":"2024-09-21T09:51:07.635475+0000","last_peered":"2024-09-21T09:51:07.635475+0000","last_clean":"2024-09-21T09:51:07.635475+0000","last_became_active":"2024-09-21T09:50:23.412745+0000","last_became_peered":"2024-09-21T09:50:23.412745+0000","last_unstale":"2024-09-21T09:51:07.635475+0000","last_undegraded":"2024-09-21T09:51:07.635475+0000","last_fullsized":"2024-09-21T09:51:07.635475+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:56:38.263798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.205212+0000","last_change":"2024-09-21T09:49:14.317504+0000","last_active":"2024-09-21T09:51:08.205212+0000","last_peered":"2024-09-21T09:51:08.205212+0000","last_clean":"2024-09-21T09:51:08.205212+0000","last_became_active":"2024-09-21T09:49:10.249697+0000","last_became_peered":"2024-09-21T09:49:10.249697+0000","last_unstale":"2024-09-21T09:51:08.205212+0000","last_undegraded":"2024-09-21T09:51:08.205212+0000","last_fullsized":"2024-09-21T09:51:08.205212+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:22.915376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039017899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203858+0000","last_change":"2024-09-21T09:50:22.400809+0000","last_active":"2024-09-21T09:51:07.203858+0000","last_peered":"2024-09-21T09:51:07.203858+0000","last_clean":"2024-09-21T09:51:07.203858+0000","last_became_active":"2024-09-21T09:50:22.400446+0000","last_became_peered":"2024-09-21T09:50:22.400446+0000","last_unstale":"2024-09-21T09:51:07.203858+0000","last_undegraded":"2024-09-21T09:51:07.203858+0000","last_fullsized":"2024-09-21T09:51:07.203858+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:53.001705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205942+0000","last_change":"2024-09-21T09:50:23.754232+0000","last_active":"2024-09-21T09:51:07.205942+0000","last_peered":"2024-09-21T09:51:07.205942+0000","last_clean":"2024-09-21T09:51:07.205942+0000","last_became_active":"2024-09-21T09:50:23.753889+0000","last_became_peered":"2024-09-21T09:50:23.753889+0000","last_unstale":"2024-09-21T09:51:07.205942+0000","last_undegraded":"2024-09-21T09:51:07.205942+0000","last_fullsized":"2024-09-21T09:51:07.205942+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:31.929579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":30,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.206695+0000","last_change":"2024-09-21T09:49:14.321622+0000","last_active":"2024-09-21T09:51:08.206695+0000","last_peered":"2024-09-21T09:51:08.206695+0000","last_clean":"2024-09-21T09:51:08.206695+0000","last_became_active":"2024-09-21T09:49:10.175207+0000","last_became_peered":"2024-09-21T09:49:10.175207+0000","last_unstale":"2024-09-21T09:51:08.206695+0000","last_undegraded":"2024-09-21T09:51:08.206695+0000","last_fullsized":"2024-09-21T09:51:08.206695+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:54.922603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091875599999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204167+0000","last_change":"2024-09-21T09:50:22.400816+0000","last_active":"2024-09-21T09:51:07.204167+0000","last_peered":"2024-09-21T09:51:07.204167+0000","last_clean":"2024-09-21T09:51:07.204167+0000","last_became_active":"2024-09-21T09:50:22.400252+0000","last_became_peered":"2024-09-21T09:50:22.400252+0000","last_unstale":"2024-09-21T09:51:07.204167+0000","last_undegraded":"2024-09-21T09:51:07.204167+0000","last_fullsized":"2024-09-21T09:51:07.204167+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:02:09.928786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.206217+0000","last_change":"2024-09-21T09:50:23.414964+0000","last_active":"2024-09-21T09:51:07.206217+0000","last_peered":"2024-09-21T09:51:07.206217+0000","last_clean":"2024-09-21T09:51:07.206217+0000","last_became_active":"2024-09-21T09:50:23.414118+0000","last_became_peered":"2024-09-21T09:50:23.414118+0000","last_unstale":"2024-09-21T09:51:07.206217+0000","last_undegraded":"2024-09-21T09:51:07.206217+0000","last_fullsized":"2024-09-21T09:51:07.206217+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:12.450540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209269+0000","last_change":"2024-09-21T09:50:22.392418+0000","last_active":"2024-09-21T09:51:07.209269+0000","last_peered":"2024-09-21T09:51:07.209269+0000","last_clean":"2024-09-21T09:51:07.209269+0000","last_became_active":"2024-09-21T09:50:22.392156+0000","last_became_peered":"2024-09-21T09:50:22.392156+0000","last_unstale":"2024-09-21T09:51:07.209269+0000","last_undegraded":"2024-09-21T09:51:07.209269+0000","last_fullsized":"2024-09-21T09:51:07.209269+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:57:16.694688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.335613+0000","last_change":"2024-09-21T09:49:14.322235+0000","last_active":"2024-09-21T09:51:08.335613+0000","last_peered":"2024-09-21T09:51:08.335613+0000","last_clean":"2024-09-21T09:51:08.335613+0000","last_became_active":"2024-09-21T09:49:10.176921+0000","last_became_peered":"2024-09-21T09:49:10.176921+0000","last_unstale":"2024-09-21T09:51:08.335613+0000","last_undegraded":"2024-09-21T09:51:08.335613+0000","last_fullsized":"2024-09-21T09:51:08.335613+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:51.116299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034704499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204269+0000","last_change":"2024-09-21T09:50:23.755246+0000","last_active":"2024-09-21T09:51:07.204269+0000","last_peered":"2024-09-21T09:51:07.204269+0000","last_clean":"2024-09-21T09:51:07.204269+0000","last_became_active":"2024-09-21T09:50:23.754824+0000","last_became_peered":"2024-09-21T09:50:23.754824+0000","last_unstale":"2024-09-21T09:51:07.204269+0000","last_undegraded":"2024-09-21T09:51:07.204269+0000","last_fullsized":"2024-09-21T09:51:07.204269+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:33:47.431559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.206243+0000","last_change":"2024-09-21T09:50:22.399629+0000","last_active":"2024-09-21T09:51:07.206243+0000","last_peered":"2024-09-21T09:51:07.206243+0000","last_clean":"2024-09-21T09:51:07.206243+0000","last_became_active":"2024-09-21T09:50:22.399348+0000","last_became_peered":"2024-09-21T09:50:22.399348+0000","last_unstale":"2024-09-21T09:51:07.206243+0000","last_undegraded":"2024-09-21T09:51:07.206243+0000","last_fullsized":"2024-09-21T09:51:07.206243+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:34.623262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.584327+0000","last_change":"2024-09-21T09:49:14.324021+0000","last_active":"2024-09-21T09:51:08.584327+0000","last_peered":"2024-09-21T09:51:08.584327+0000","last_clean":"2024-09-21T09:51:08.584327+0000","last_became_active":"2024-09-21T09:49:10.249765+0000","last_became_peered":"2024-09-21T09:49:10.249765+0000","last_unstale":"2024-09-21T09:51:08.584327+0000","last_undegraded":"2024-09-21T09:51:08.584327+0000","last_fullsized":"2024-09-21T09:51:08.584327+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:13.890155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031296500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635297+0000","last_change":"2024-09-21T09:50:23.755064+0000","last_active":"2024-09-21T09:51:07.635297+0000","last_peered":"2024-09-21T09:51:07.635297+0000","last_clean":"2024-09-21T09:51:07.635297+0000","last_became_active":"2024-09-21T09:50:23.754728+0000","last_became_peered":"2024-09-21T09:50:23.754728+0000","last_unstale":"2024-09-21T09:51:07.635297+0000","last_undegraded":"2024-09-21T09:51:07.635297+0000","last_fullsized":"2024-09-21T09:51:07.635297+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:43:33.571075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203896+0000","last_change":"2024-09-21T09:50:22.400077+0000","last_active":"2024-09-21T09:51:07.203896+0000","last_peered":"2024-09-21T09:51:07.203896+0000","last_clean":"2024-09-21T09:51:07.203896+0000","last_became_active":"2024-09-21T09:50:22.399639+0000","last_became_peered":"2024-09-21T09:50:22.399639+0000","last_unstale":"2024-09-21T09:51:07.203896+0000","last_undegraded":"2024-09-21T09:51:07.203896+0000","last_fullsized":"2024-09-21T09:51:07.203896+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:46.929718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.335782+0000","last_change":"2024-09-21T09:49:14.322228+0000","last_active":"2024-09-21T09:51:08.335782+0000","last_peered":"2024-09-21T09:51:08.335782+0000","last_clean":"2024-09-21T09:51:08.335782+0000","last_became_active":"2024-09-21T09:49:10.249779+0000","last_became_peered":"2024-09-21T09:49:10.249779+0000","last_unstale":"2024-09-21T09:51:08.335782+0000","last_undegraded":"2024-09-21T09:51:08.335782+0000","last_fullsized":"2024-09-21T09:51:08.335782+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:44.504161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033889199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204001+0000","last_change":"2024-09-21T09:50:22.931457+0000","last_active":"2024-09-21T09:51:07.204001+0000","last_peered":"2024-09-21T09:51:07.204001+0000","last_clean":"2024-09-21T09:51:07.204001+0000","last_became_active":"2024-09-21T09:50:22.931121+0000","last_became_peered":"2024-09-21T09:50:22.931121+0000","last_unstale":"2024-09-21T09:51:07.204001+0000","last_undegraded":"2024-09-21T09:51:07.204001+0000","last_fullsized":"2024-09-21T09:51:07.204001+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:18.905294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635552+0000","last_change":"2024-09-21T09:50:22.399135+0000","last_active":"2024-09-21T09:51:07.635552+0000","last_peered":"2024-09-21T09:51:07.635552+0000","last_clean":"2024-09-21T09:51:07.635552+0000","last_became_active":"2024-09-21T09:50:22.398731+0000","last_became_peered":"2024-09-21T09:50:22.398731+0000","last_unstale":"2024-09-21T09:51:07.635552+0000","last_undegraded":"2024-09-21T09:51:07.635552+0000","last_fullsized":"2024-09-21T09:51:07.635552+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:09.733443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.204796+0000","last_change":"2024-09-21T09:49:14.322989+0000","last_active":"2024-09-21T09:51:08.204796+0000","last_peered":"2024-09-21T09:51:08.204796+0000","last_clean":"2024-09-21T09:51:08.204796+0000","last_became_active":"2024-09-21T09:49:10.176476+0000","last_became_peered":"2024-09-21T09:49:10.176476+0000","last_unstale":"2024-09-21T09:51:08.204796+0000","last_undegraded":"2024-09-21T09:51:08.204796+0000","last_fullsized":"2024-09-21T09:51:08.204796+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:03:19.334046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058244600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203594+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T09:51:07.203594+0000","last_peered":"2024-09-21T09:51:07.203594+0000","last_clean":"2024-09-21T09:51:07.203594+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T09:51:07.203594+0000","last_undegraded":"2024-09-21T09:51:07.203594+0000","last_fullsized":"2024-09-21T09:51:07.203594+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204006+0000","last_change":"2024-09-21T09:50:23.756473+0000","last_active":"2024-09-21T09:51:07.204006+0000","last_peered":"2024-09-21T09:51:07.204006+0000","last_clean":"2024-09-21T09:51:07.204006+0000","last_became_active":"2024-09-21T09:50:23.755051+0000","last_became_peered":"2024-09-21T09:50:23.755051+0000","last_unstale":"2024-09-21T09:51:07.204006+0000","last_undegraded":"2024-09-21T09:51:07.204006+0000","last_fullsized":"2024-09-21T09:51:07.204006+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:51:14.382136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203766+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T09:51:07.203766+0000","last_peered":"2024-09-21T09:51:07.203766+0000","last_clean":"2024-09-21T09:51:07.203766+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T09:51:07.203766+0000","last_undegraded":"2024-09-21T09:51:07.203766+0000","last_fullsized":"2024-09-21T09:51:07.203766+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.207885+0000","last_change":"2024-09-21T09:51:08.207885+0000","last_active":"2024-09-21T09:51:08.207885+0000","last_peered":"2024-09-21T09:51:08.207885+0000","last_clean":"2024-09-21T09:51:08.207885+0000","last_became_active":"2024-09-21T09:51:08.207526+0000","last_became_peered":"2024-09-21T09:51:08.207526+0000","last_unstale":"2024-09-21T09:51:08.207885+0000","last_undegraded":"2024-09-21T09:51:08.207885+0000","last_fullsized":"2024-09-21T09:51:08.207885+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:31:02.839035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635957+0000","last_change":"2024-09-21T09:50:22.402043+0000","last_active":"2024-09-21T09:51:07.635957+0000","last_peered":"2024-09-21T09:51:07.635957+0000","last_clean":"2024-09-21T09:51:07.635957+0000","last_became_active":"2024-09-21T09:50:22.401686+0000","last_became_peered":"2024-09-21T09:50:22.401686+0000","last_unstale":"2024-09-21T09:51:07.635957+0000","last_undegraded":"2024-09-21T09:51:07.635957+0000","last_fullsized":"2024-09-21T09:51:07.635957+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:28.871212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209109+0000","last_change":"2024-09-21T09:50:22.394664+0000","last_active":"2024-09-21T09:51:07.209109+0000","last_peered":"2024-09-21T09:51:07.209109+0000","last_clean":"2024-09-21T09:51:07.209109+0000","last_became_active":"2024-09-21T09:50:22.394338+0000","last_became_peered":"2024-09-21T09:50:22.394338+0000","last_unstale":"2024-09-21T09:51:07.209109+0000","last_undegraded":"2024-09-21T09:51:07.209109+0000","last_fullsized":"2024-09-21T09:51:07.209109+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:09.429993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203326+0000","last_change":"2024-09-21T09:50:22.929959+0000","last_active":"2024-09-21T09:51:07.203326+0000","last_peered":"2024-09-21T09:51:07.203326+0000","last_clean":"2024-09-21T09:51:07.203326+0000","last_became_active":"2024-09-21T09:50:22.929655+0000","last_became_peered":"2024-09-21T09:50:22.929655+0000","last_unstale":"2024-09-21T09:51:07.203326+0000","last_undegraded":"2024-09-21T09:51:07.203326+0000","last_fullsized":"2024-09-21T09:51:07.203326+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:57.365409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635610+0000","last_change":"2024-09-21T09:50:22.398412+0000","last_active":"2024-09-21T09:51:07.635610+0000","last_peered":"2024-09-21T09:51:07.635610+0000","last_clean":"2024-09-21T09:51:07.635610+0000","last_became_active":"2024-09-21T09:50:22.398108+0000","last_became_peered":"2024-09-21T09:50:22.398108+0000","last_unstale":"2024-09-21T09:51:07.635610+0000","last_undegraded":"2024-09-21T09:51:07.635610+0000","last_fullsized":"2024-09-21T09:51:07.635610+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:11.058999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.206483+0000","last_change":"2024-09-21T09:51:08.206483+0000","last_active":"2024-09-21T09:51:08.206483+0000","last_peered":"2024-09-21T09:51:08.206483+0000","last_clean":"2024-09-21T09:51:08.206483+0000","last_became_active":"2024-09-21T09:51:08.206154+0000","last_became_peered":"2024-09-21T09:51:08.206154+0000","last_unstale":"2024-09-21T09:51:08.206483+0000","last_undegraded":"2024-09-21T09:51:08.206483+0000","last_fullsized":"2024-09-21T09:51:08.206483+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:07:16.789019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:51:08.208430+0000","last_change":"2024-09-21T09:51:08.208430+0000","last_active":"2024-09-21T09:51:08.208430+0000","last_peered":"2024-09-21T09:51:08.208430+0000","last_clean":"2024-09-21T09:51:08.208430+0000","last_became_active":"2024-09-21T09:51:08.207843+0000","last_became_peered":"2024-09-21T09:51:08.207843+0000","last_unstale":"2024-09-21T09:51:08.208430+0000","last_undegraded":"2024-09-21T09:51:08.208430+0000","last_fullsized":"2024-09-21T09:51:08.208430+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:38.622103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204085+0000","last_change":"2024-09-21T09:50:22.402812+0000","last_active":"2024-09-21T09:51:07.204085+0000","last_peered":"2024-09-21T09:51:07.204085+0000","last_clean":"2024-09-21T09:51:07.204085+0000","last_became_active":"2024-09-21T09:50:22.402273+0000","last_became_peered":"2024-09-21T09:50:22.402273+0000","last_unstale":"2024-09-21T09:51:07.204085+0000","last_undegraded":"2024-09-21T09:51:07.204085+0000","last_fullsized":"2024-09-21T09:51:07.204085+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:40.454290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635438+0000","last_change":"2024-09-21T09:50:23.417280+0000","last_active":"2024-09-21T09:51:07.635438+0000","last_peered":"2024-09-21T09:51:07.635438+0000","last_clean":"2024-09-21T09:51:07.635438+0000","last_became_active":"2024-09-21T09:50:23.417002+0000","last_became_peered":"2024-09-21T09:50:23.417002+0000","last_unstale":"2024-09-21T09:51:07.635438+0000","last_undegraded":"2024-09-21T09:51:07.635438+0000","last_fullsized":"2024-09-21T09:51:07.635438+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:49.501587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.204064+0000","last_change":"2024-09-21T09:50:22.405044+0000","last_active":"2024-09-21T09:51:07.204064+0000","last_peered":"2024-09-21T09:51:07.204064+0000","last_clean":"2024-09-21T09:51:07.204064+0000","last_became_active":"2024-09-21T09:50:22.403898+0000","last_became_peered":"2024-09-21T09:50:22.403898+0000","last_unstale":"2024-09-21T09:51:07.204064+0000","last_undegraded":"2024-09-21T09:51:07.204064+0000","last_fullsized":"2024-09-21T09:51:07.204064+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:23:33.971849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203767+0000","last_change":"2024-09-21T09:50:22.820199+0000","last_active":"2024-09-21T09:51:07.203767+0000","last_peered":"2024-09-21T09:51:07.203767+0000","last_clean":"2024-09-21T09:51:07.203767+0000","last_became_active":"2024-09-21T09:50:22.819651+0000","last_became_peered":"2024-09-21T09:50:22.819651+0000","last_unstale":"2024-09-21T09:51:07.203767+0000","last_undegraded":"2024-09-21T09:51:07.203767+0000","last_fullsized":"2024-09-21T09:51:07.203767+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:16.499801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203664+0000","last_change":"2024-09-21T09:50:22.404585+0000","last_active":"2024-09-21T09:51:07.203664+0000","last_peered":"2024-09-21T09:51:07.203664+0000","last_clean":"2024-09-21T09:51:07.203664+0000","last_became_active":"2024-09-21T09:50:22.404279+0000","last_became_peered":"2024-09-21T09:50:22.404279+0000","last_unstale":"2024-09-21T09:51:07.203664+0000","last_undegraded":"2024-09-21T09:51:07.203664+0000","last_fullsized":"2024-09-21T09:51:07.203664+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:29:22.909031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.207498+0000","last_change":"2024-09-21T09:50:23.415908+0000","last_active":"2024-09-21T09:51:07.207498+0000","last_peered":"2024-09-21T09:51:07.207498+0000","last_clean":"2024-09-21T09:51:07.207498+0000","last_became_active":"2024-09-21T09:50:23.415594+0000","last_became_peered":"2024-09-21T09:50:23.415594+0000","last_unstale":"2024-09-21T09:51:07.207498+0000","last_undegraded":"2024-09-21T09:51:07.207498+0000","last_fullsized":"2024-09-21T09:51:07.207498+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:22.373294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635406+0000","last_change":"2024-09-21T09:50:22.406273+0000","last_active":"2024-09-21T09:51:07.635406+0000","last_peered":"2024-09-21T09:51:07.635406+0000","last_clean":"2024-09-21T09:51:07.635406+0000","last_became_active":"2024-09-21T09:50:22.405984+0000","last_became_peered":"2024-09-21T09:50:22.405984+0000","last_unstale":"2024-09-21T09:51:07.635406+0000","last_undegraded":"2024-09-21T09:51:07.635406+0000","last_fullsized":"2024-09-21T09:51:07.635406+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:39.609068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203973+0000","last_change":"2024-09-21T09:50:23.414339+0000","last_active":"2024-09-21T09:51:07.203973+0000","last_peered":"2024-09-21T09:51:07.203973+0000","last_clean":"2024-09-21T09:51:07.203973+0000","last_became_active":"2024-09-21T09:50:23.414046+0000","last_became_peered":"2024-09-21T09:50:23.414046+0000","last_unstale":"2024-09-21T09:51:07.203973+0000","last_undegraded":"2024-09-21T09:51:07.203973+0000","last_fullsized":"2024-09-21T09:51:07.203973+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:14.632255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209185+0000","last_change":"2024-09-21T09:50:22.397124+0000","last_active":"2024-09-21T09:51:07.209185+0000","last_peered":"2024-09-21T09:51:07.209185+0000","last_clean":"2024-09-21T09:51:07.209185+0000","last_became_active":"2024-09-21T09:50:22.396873+0000","last_became_peered":"2024-09-21T09:50:22.396873+0000","last_unstale":"2024-09-21T09:51:07.209185+0000","last_undegraded":"2024-09-21T09:51:07.209185+0000","last_fullsized":"2024-09-21T09:51:07.209185+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:46:54.625943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203686+0000","last_change":"2024-09-21T09:50:23.416269+0000","last_active":"2024-09-21T09:51:07.203686+0000","last_peered":"2024-09-21T09:51:07.203686+0000","last_clean":"2024-09-21T09:51:07.203686+0000","last_became_active":"2024-09-21T09:50:23.415890+0000","last_became_peered":"2024-09-21T09:50:23.415890+0000","last_unstale":"2024-09-21T09:51:07.203686+0000","last_undegraded":"2024-09-21T09:51:07.203686+0000","last_fullsized":"2024-09-21T09:51:07.203686+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:39.754711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.209251+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T09:51:07.209251+0000","last_peered":"2024-09-21T09:51:07.209251+0000","last_clean":"2024-09-21T09:51:07.209251+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T09:51:07.209251+0000","last_undegraded":"2024-09-21T09:51:07.209251+0000","last_fullsized":"2024-09-21T09:51:07.209251+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203418+0000","last_change":"2024-09-21T09:50:22.929558+0000","last_active":"2024-09-21T09:51:07.203418+0000","last_peered":"2024-09-21T09:51:07.203418+0000","last_clean":"2024-09-21T09:51:07.203418+0000","last_became_active":"2024-09-21T09:50:22.929186+0000","last_became_peered":"2024-09-21T09:50:22.929186+0000","last_unstale":"2024-09-21T09:51:07.203418+0000","last_undegraded":"2024-09-21T09:51:07.203418+0000","last_fullsized":"2024-09-21T09:51:07.203418+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:15:32.908972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.205126+0000","last_change":"2024-09-21T09:50:22.394608+0000","last_active":"2024-09-21T09:51:07.205126+0000","last_peered":"2024-09-21T09:51:07.205126+0000","last_clean":"2024-09-21T09:51:07.205126+0000","last_became_active":"2024-09-21T09:50:22.394260+0000","last_became_peered":"2024-09-21T09:50:22.394260+0000","last_unstale":"2024-09-21T09:51:07.205126+0000","last_undegraded":"2024-09-21T09:51:07.205126+0000","last_fullsized":"2024-09-21T09:51:07.205126+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:34:00.389844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.203407+0000","last_change":"2024-09-21T09:50:23.756088+0000","last_active":"2024-09-21T09:51:07.203407+0000","last_peered":"2024-09-21T09:51:07.203407+0000","last_clean":"2024-09-21T09:51:07.203407+0000","last_became_active":"2024-09-21T09:50:23.755515+0000","last_became_peered":"2024-09-21T09:50:23.755515+0000","last_unstale":"2024-09-21T09:51:07.203407+0000","last_undegraded":"2024-09-21T09:51:07.203407+0000","last_fullsized":"2024-09-21T09:51:07.203407+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:35.236821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:51:07.635565+0000","last_change":"2024-09-21T09:50:22.401825+0000","last_active":"2024-09-21T09:51:07.635565+0000","last_peered":"2024-09-21T09:51:07.635565+0000","last_clean":"2024-09-21T09:51:07.635565+0000","last_became_active":"2024-09-21T09:50:22.401550+0000","last_became_peered":"2024-09-21T09:50:22.401550+0000","last_unstale":"2024-09-21T09:51:07.635565+0000","last_undegraded":"2024-09-21T09:51:07.635565+0000","last_fullsized":"2024-09-21T09:51:07.635565+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:08.792265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234020,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28368,"kb_used_data":1344,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716784,"statfs":{"total":95995035648,"available":95965986816,"internally_reserved":0,"allocated":1376256,"data_stored":925421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.36299999999999999,"5min":0.312,"15min":0.312},"max":{"1min":0.71499999999999997,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.40000000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.496},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.53400000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.41299999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.58099999999999996,"15min":0.58899999999999997},"min":{"1min":0.34499999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.755,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.438}]},{"osd":3,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.755,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59399999999999997,"15min":0.59499999999999997},"min":{"1min":0.436,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76900000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.60599999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.48199999999999998,"15min":0.48299999999999998},"min":{"1min":0.32900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.69899999999999995,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.62},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.52300000000000002,"15min":0.52200000000000002},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.73199999999999998,"5min":0.871,"15min":0.871},"last":0.57999999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.81000000000000005,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.54400000000000004,"15min":0.53800000000000003},"min":{"1min":0.38300000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.47699999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.57399999999999995,"15min":0.56999999999999995},"min":{"1min":0.38200000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79400000000000004,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40999999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77000000000000002,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.63100000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495658,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28364,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716788,"statfs":{"total":95995035648,"available":95965990912,"internally_reserved":0,"allocated":1372160,"data_stored":925625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51500000000000001,"15min":0.50700000000000001},"min":{"1min":0.373,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.53000000000000003,"15min":0.51200000000000001},"min":{"1min":0.40000000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76900000000000002,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.44700000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.59899999999999998,"15min":0.54700000000000004},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78100000000000003,"5min":6.008,"15min":6.008},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54000000000000004,"15min":0.53100000000000003},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.59099999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59299999999999997,"15min":0.53700000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.80700000000000005,"5min":6.0309999999999997,"15min":6.0309999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.40500000000000003,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.68000000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.65800000000000003,"15min":0.58199999999999996},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.481,"5min":10.481,"15min":10.481},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.056,"5min":0.72299999999999998,"15min":0.60699999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.518000000000001,"5min":10.518000000000001,"15min":10.518000000000001},"last":0.66500000000000004}]},{"osd":4,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.437,"15min":0.42899999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.622,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.44700000000000001,"15min":0.42899999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.69299999999999995,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.64600000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.45200000000000001,"15min":0.44},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.628},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.47999999999999998,"15min":0.46300000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.75600000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:50:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.54600000000000004,"15min":0.53900000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.50700000000000001,"15min":0.505},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.69699999999999995}]}]},{"osd":5,"up_from":38,"seq":163208757295,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":770048,"data_stored":334275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.57399999999999995,"15min":0.56100000000000005},"min":{"1min":0.34100000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.536,"5min":2.536,"15min":2.536},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.54400000000000004,"15min":0.55300000000000005},"min":{"1min":0.35499999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.73899999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.66100000000000003}]},{"osd":1,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.72199999999999998,"15min":0.79100000000000004},"min":{"1min":0.38,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71999999999999997,"5min":7.2190000000000003,"15min":7.2190000000000003},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.77100000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.60999999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.70099999999999996,"15min":0.77500000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.82199999999999995,"5min":7.1619999999999999,"15min":7.1619999999999999},"last":0.62},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59199999999999997,"15min":0.60199999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.75800000000000001,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.63600000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.55600000000000005,"15min":0.56399999999999995},"min":{"1min":0.376,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.67800000000000005,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.753,"15min":0.82499999999999996},"min":{"1min":0.42999999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78300000000000003,"5min":7.181,"15min":7.181},"last":0.71199999999999997}]},{"osd":4,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.437,"15min":0.433},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.60099999999999998,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.46600000000000003,"15min":0.46100000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.63,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.65000000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:51:16 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.318,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.70599999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.753,"5min":1.179,"15min":1.179},"last":0.52600000000000002}]},{"osd":7,"last update":"Sat Sep 21 09:50:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.59599999999999997,"15min":0.61399999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70199999999999996,"5min":1.24,"15min":1.24},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.54200000000000004,"15min":0.55100000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68500000000000005,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.57399999999999995}]}]},{"osd":4,"up_from":30,"seq":128849018931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31880,"kb_used_data":764,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713272,"statfs":{"total":95995035648,"available":95962390528,"internally_reserved":0,"allocated":782336,"data_stored":335314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.72699999999999998,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.70899999999999996,"15min":0.77900000000000003},"min":{"1min":0.39400000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.71699999999999997,"5min":5.4029999999999996,"15min":5.4029999999999996},"last":0.47299999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.38900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.80800000000000005,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.495,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.39700000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.67400000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52100000000000002,"15min":0.52400000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.60999999999999999,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.499},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.58299999999999996,"15min":0.58399999999999996},"min":{"1min":0.41399999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.72099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.55000000000000004}]},{"osd":3,"last update":"Sat Sep 21 09:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.80500000000000005,"15min":0.80900000000000005},"min":{"1min":0.45200000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.747,"5min":8.0060000000000002,"15min":8.0060000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.72799999999999998,"15min":0.78900000000000003},"min":{"1min":0.45900000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80100000000000005,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:50:50 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.41099999999999998,"15min":0.38800000000000001},"min":{"1min":0.311,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67400000000000004,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.435,"15min":0.40200000000000002},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69699999999999995,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:51:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70399999999999996,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.379},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52200000000000002,"15min":0.51400000000000001},"min":{"1min":0.36099999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.749,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.53100000000000003}]},{"osd":7,"last update":"Sat Sep 21 09:50:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56200000000000006,"15min":0.56699999999999995},"min":{"1min":0.374,"5min":0.316,"15min":0.316},"max":{"1min":0.67400000000000004,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.44},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.58699999999999997,"15min":0.59499999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.74199999999999999,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45000000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182456,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":333694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:51:09 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.38800000000000001,"15min":0.39300000000000002},"min":{"1min":0.215,"5min":0.185,"15min":0.185},"max":{"1min":0.65900000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.40699999999999997,"15min":0.432},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.60399999999999998,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.39900000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:51:09 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.42499999999999999,"15min":0.44900000000000001},"min":{"1min":0.317,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.48699999999999999,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.34399999999999997,"5min":0.39100000000000001,"15min":0.40699999999999997},"min":{"1min":0.23599999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.58999999999999997,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.29999999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:51:09 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.437,"15min":0.45600000000000002},"min":{"1min":0.29799999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.48399999999999999,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.42399999999999999,"15min":0.45200000000000001},"min":{"1min":0.26000000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.46300000000000002,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:50:31 2024","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":0.746,"15min":0.65300000000000002},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":15.769,"5min":15.769,"15min":15.769},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.433,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81499999999999995,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.56799999999999995}]},{"osd":5,"last update":"Sat Sep 21 09:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.59799999999999998,"15min":0.61399999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.67900000000000005,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.59099999999999997,"15min":0.60299999999999998},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.37},"max":{"1min":0.76800000000000002,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:51:15 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.76100000000000001,"15min":0.66600000000000004},"min":{"1min":0.41799999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.72299999999999998,"5min":15.784000000000001,"15min":15.784000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.59799999999999998,"15min":0.60399999999999998},"min":{"1min":0.36699999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.83699999999999997,"5min":0.877,"15min":0.877},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Sep 21 09:50:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79300000000000004,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.60299999999999998,"15min":0.60799999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.73999999999999999,"5min":0.755,"15min":0.755},"last":0.56100000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31868,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713284,"statfs":{"total":95995035648,"available":95962402816,"internally_reserved":0,"allocated":765952,"data_stored":334220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:44 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.35699999999999998,"15min":0.35299999999999998},"min":{"1min":0.222,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.51900000000000002,"5min":0.879,"15min":0.879},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.37,"15min":0.36399999999999999},"min":{"1min":0.251,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.51000000000000001,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52500000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:50:44 2024","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.36799999999999999,"15min":0.36499999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.47799999999999998,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.38500000000000001,"15min":0.38100000000000001},"min":{"1min":0.28100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":0.50700000000000001,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.504}]},{"osd":3,"last update":"Sat Sep 21 09:51:10 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.47799999999999998,"15min":0.48399999999999999},"min":{"1min":0.33500000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.147,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.443,"15min":0.44700000000000001},"min":{"1min":0.318,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.57399999999999995,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.60699999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:50:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.59399999999999997,"15min":0.59699999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.59599999999999997,"15min":0.59699999999999998},"min":{"1min":0.46100000000000002,"5min":0.373,"15min":0.373},"max":{"1min":0.74399999999999999,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.71599999999999997}]},{"osd":5,"last update":"Sat Sep 21 09:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.57699999999999996,"15min":0.58499999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76300000000000001,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.65000000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:51:19 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.57399999999999995,"15min":0.56599999999999995},"min":{"1min":0.38400000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.63800000000000001,"15min":0.626},"min":{"1min":0.47099999999999997,"5min":0.376,"15min":0.376},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.74199999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:50:38 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.629,"15min":0.63800000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.75700000000000001,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.61299999999999999,"15min":0.61499999999999999},"min":{"1min":0.42799999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.79100000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27760,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717392,"statfs":{"total":95995035648,"available":95966609408,"internally_reserved":0,"allocated":757760,"data_stored":333350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:50:19 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.33400000000000002,"15min":0.317},"min":{"1min":0.23300000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.55000000000000004,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.34799999999999998,"15min":0.32800000000000001},"min":{"1min":0.23699999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.50900000000000001,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.317}]},{"osd":2,"last update":"Sat Sep 21 09:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42799999999999999,"15min":0.41099999999999998},"min":{"1min":0.33500000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.54600000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.432},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.39000000000000001,"15min":0.36699999999999999},"min":{"1min":0.255,"5min":0.222,"15min":0.222},"max":{"1min":0.45600000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:51:10 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.39900000000000002,"15min":0.39100000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.48099999999999998,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.42999999999999999,"15min":0.42599999999999999},"min":{"1min":0.31900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.51100000000000001,"5min":0.63,"15min":0.63},"last":0.46600000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:50:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.35999999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.80900000000000005,"5min":0.88,"15min":0.88},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56799999999999995,"15min":0.56599999999999995},"min":{"1min":0.39500000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.498}]},{"osd":5,"last update":"Sat Sep 21 09:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57699999999999996,"15min":0.58299999999999996},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.45700000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:51:14 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.60899999999999999,"15min":0.61399999999999999},"min":{"1min":0.44500000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.79300000000000004,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.60699999999999998,"15min":0.60999999999999999},"min":{"1min":0.40600000000000003,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.86599999999999999,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.50600000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:50:36 2024","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":0.68799999999999994,"15min":0.61499999999999999},"min":{"1min":0.46600000000000003,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":10.279,"5min":10.279,"15min":10.279},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59899999999999998,"15min":0.59799999999999998},"min":{"1min":0.38,"5min":0.378,"15min":0.378},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.51600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32448,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712704,"statfs":{"total":95995035648,"available":95961808896,"internally_reserved":0,"allocated":1359872,"data_stored":924576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.34499999999999997,"15min":0.33500000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":0.54700000000000004,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.35099999999999998,"15min":0.33500000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.55100000000000005,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.39400000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:50:46 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.38900000000000001,"15min":0.38400000000000001},"min":{"1min":0.252,"5min":0.246,"15min":0.246},"max":{"1min":0.54800000000000004,"5min":0.622,"15min":0.622},"last":0.441},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.432,"15min":0.433},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60799999999999998,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:51:08 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.42099999999999999,"15min":0.435},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.53200000000000003,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.5},{"interface":"front","average":{"1min":0.38,"5min":0.436,"15min":0.46500000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.65500000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.45500000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.58199999999999996,"15min":0.60099999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.88,"5min":1.036,"15min":1.036},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.62,"15min":0.61599999999999999},"min":{"1min":0.40300000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.80600000000000005,"5min":4.4210000000000003,"15min":4.4210000000000003},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:50:51 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.61799999999999999,"15min":0.60799999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":4.8019999999999996,"5min":4.8019999999999996,"15min":4.8019999999999996},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.60599999999999998,"15min":0.63300000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81599999999999995,"5min":1.254,"15min":1.254},"last":0.77000000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:51:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.57999999999999996,"15min":0.58999999999999997},"min":{"1min":0.40999999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70599999999999996,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.71599999999999997,"15min":0.65700000000000003},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.80400000000000005,"5min":13.663,"15min":13.663},"last":0.73399999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.61099999999999999,"15min":0.62},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89400000000000002,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.64300000000000002,"15min":0.65300000000000002},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90800000000000003,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.68100000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:51:24.200 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:51:24.200 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:51:24.200 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:51:24.200 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:51:24.408 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:24.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:24 smithi032 ceph-mon[29191]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:51:24.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:24 smithi032 ceph-mon[40061]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:51:24.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:24 smithi045 ceph-mon[33087]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:51:25.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:25 smithi032 ceph-mon[29191]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:25 smithi032 ceph-mon[40061]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:25.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:25 smithi045 ceph-mon[33087]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:26.306 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T09:51:26.307 INFO:teuthology.orchestra.run.smithi032.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:51:26.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:26 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1902858102' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:51:26.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:26 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1902858102' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:51:26.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:26 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1902858102' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:51:27.203 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:51:27.204 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:51:27.306 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-21T09:51:27.306 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T09:51:27.532 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T09:51:27.533 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:51:27.533 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-21T09:51:27.537 INFO:tasks.workunit:Pulling workunits from ref reef 2024-09-21T09:51:27.538 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-21T09:51:27.538 DEBUG:teuthology.orchestra.run.smithi032:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:51:27.538 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:51:27.538 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T09:51:27.539 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi032.front.sepia.ceph.com 2024-09-21T09:51:27.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-21T09:51:27.539 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:27 smithi032 ceph-mon[29191]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:27.540 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:27 smithi032 ceph-mon[40061]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:51:27.560 INFO:teuthology.orchestra.run.smithi032.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-21T09:51:27.560 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:51:27.617 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-21T09:51:27.617 DEBUG:teuthology.orchestra.run.smithi032:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-21T09:51:27.673 INFO:tasks.workunit:timeout=3h 2024-09-21T09:51:27.673 INFO:tasks.workunit:cleanup=True 2024-09-21T09:51:27.674 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-21T09:51:27.730 INFO:tasks.workunit.client.0.smithi032.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-21T09:51:27.764 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:27.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:27 smithi045 ceph-mon[33087]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:29.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:29 smithi032 ceph-mon[29191]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:29.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:29 smithi032 ceph-mon[40061]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:29.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:29 smithi045 ceph-mon[33087]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:29.893 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:51:29] ENGINE Bus STOPPING 2024-09-21T09:51:29.893 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:51:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:51:29.893 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:51:29] ENGINE Bus STOPPED 2024-09-21T09:51:29.894 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:51:29] ENGINE Bus STARTING 2024-09-21T09:51:29.894 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:51:29] ENGINE Serving on http://:::9283 2024-09-21T09:51:29.894 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:51:29] ENGINE Bus STARTED 2024-09-21T09:51:30.257 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-21T09:51:30.503 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:30.881 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3729835359' entity='client.admin' 2024-09-21T09:51:30.882 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:30.882 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:30.882 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:30.882 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:30.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3729835359' entity='client.admin' 2024-09-21T09:51:30.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:30.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:30.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:30.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:30 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:31.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:30 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3729835359' entity='client.admin' 2024-09-21T09:51:31.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:30 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:30 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:30 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:30 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:51:31.844 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:31 smithi032 ceph-mon[40061]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:31.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:31 smithi032 ceph-mon[29191]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:32.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:31 smithi045 ceph-mon[33087]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:33.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:33 smithi032 ceph-mon[29191]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:33.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:33 smithi032 ceph-mon[40061]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:33.523 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for mgr daemons to upgrade' 2024-09-21T09:51:33.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:33 smithi045 ceph-mon[33087]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:33.736 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:33.857 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:33 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:51:33] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2024-09-21T09:51:33.897 DEBUG:teuthology.orchestra.run.smithi032:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-21T09:51:33.938 INFO:tasks.workunit.client.0.smithi032.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-21T09:51:33.940 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T09:51:33.940 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-21T09:51:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:35 smithi045 ceph-mon[33087]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:35.612 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:35 smithi032 ceph-mon[29191]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:35.613 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:35 smithi032 ceph-mon[40061]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:35.953 INFO:teuthology.orchestra.run.smithi032.stdout:wait for mgr daemons to upgrade 2024-09-21T09:51:36.033 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-21T09:51:36.175 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-21T09:51:36.218 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T09:51:36.219 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T09:51:36.219 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-21T09:51:36.259 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T09:51:36.263 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T09:51:36.263 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-21T09:51:36.320 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2024-09-21T09:51:36.321 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-09-21T09:51:36.321 DEBUG:teuthology.orchestra.run.smithi032:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2024-09-21T09:51:36.384 INFO:tasks.workunit.client.0.smithi032.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-09-21T09:51:36.384 INFO:tasks.workunit.client.0.smithi032.stderr:+ RBD_FORCE_ALLOW_V1=1 2024-09-21T09:51:36.385 INFO:tasks.workunit.client.0.smithi032.stderr:+ TMPDIR=/tmp/rbd_import_export_79662 2024-09-21T09:51:36.385 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm -rf /tmp/rbd_import_export_79662 2024-09-21T09:51:36.385 INFO:tasks.workunit.client.0.smithi032.stderr:+ mkdir /tmp/rbd_import_export_79662 2024-09-21T09:51:36.386 INFO:tasks.workunit.client.0.smithi032.stderr:+ trap 'rm -rf /tmp/rbd_import_export_79662' INT TERM EXIT 2024-09-21T09:51:36.386 INFO:tasks.workunit.client.0.smithi032.stderr:+ mkdir foo.79662 2024-09-21T09:51:36.387 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import foo.79662 foo.dir 2024-09-21T09:51:36.471 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: cannot import a directory 2024-09-21T09:51:36.471 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 0% complete...failed. 2024-09-21T09:51:36.471 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: import failed: (21) Is a directory 2024-09-21T09:51:36.474 INFO:tasks.workunit.client.0.smithi032.stderr:+ true 2024-09-21T09:51:36.474 INFO:tasks.workunit.client.0.smithi032.stderr:+ rmdir foo.79662 2024-09-21T09:51:36.507 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_79662/img bs=1k count=1 seek=10 2024-09-21T09:51:36.508 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records in 2024-09-21T09:51:36.508 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records out 2024-09-21T09:51:36.508 INFO:tasks.workunit.client.0.smithi032.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.4932e-05 s, 22.8 MB/s 2024-09-21T09:51:36.509 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_79662/img bs=1k count=10 seek=100 2024-09-21T09:51:36.509 INFO:tasks.workunit.client.0.smithi032.stderr:10+0 records in 2024-09-21T09:51:36.509 INFO:tasks.workunit.client.0.smithi032.stderr:10+0 records out 2024-09-21T09:51:36.509 INFO:tasks.workunit.client.0.smithi032.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.5861e-05 s, 183 MB/s 2024-09-21T09:51:36.509 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_79662/img bs=1k count=100 seek=1000 2024-09-21T09:51:36.510 INFO:tasks.workunit.client.0.smithi032.stderr:60+1 records in 2024-09-21T09:51:36.510 INFO:tasks.workunit.client.0.smithi032.stderr:60+1 records out 2024-09-21T09:51:36.510 INFO:tasks.workunit.client.0.smithi032.stderr:61640 bytes (62 kB, 60 KiB) copied, 0.000180634 s, 341 MB/s 2024-09-21T09:51:36.510 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_79662/img bs=1k seek=10000 2024-09-21T09:51:36.511 INFO:tasks.workunit.client.0.smithi032.stderr:137+1 records in 2024-09-21T09:51:36.511 INFO:tasks.workunit.client.0.smithi032.stderr:137+1 records out 2024-09-21T09:51:36.511 INFO:tasks.workunit.client.0.smithi032.stderr:140928 bytes (141 kB, 138 KiB) copied, 0.000551643 s, 255 MB/s 2024-09-21T09:51:36.512 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_79662/img bs=1k seek=100000 2024-09-21T09:51:36.512 INFO:tasks.workunit.client.0.smithi032.stderr:60+1 records in 2024-09-21T09:51:36.512 INFO:tasks.workunit.client.0.smithi032.stderr:60+1 records out 2024-09-21T09:51:36.512 INFO:tasks.workunit.client.0.smithi032.stderr:61592 bytes (62 kB, 60 KiB) copied, 0.000181662 s, 339 MB/s 2024-09-21T09:51:36.513 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_79662/img bs=1k seek=1000000 2024-09-21T09:51:36.513 INFO:tasks.workunit.client.0.smithi032.stderr:154+1 records in 2024-09-21T09:51:36.514 INFO:tasks.workunit.client.0.smithi032.stderr:154+1 records out 2024-09-21T09:51:36.514 INFO:tasks.workunit.client.0.smithi032.stderr:158176 bytes (158 kB, 154 KiB) copied, 0.000381934 s, 414 MB/s 2024-09-21T09:51:36.514 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm testimg 2024-09-21T09:51:36.556 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 0% complete...failed. 2024-09-21T09:51:36.556 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: delete error: (2) No such file or directory 2024-09-21T09:51:36.559 INFO:tasks.workunit.client.0.smithi032.stderr:+ true 2024-09-21T09:51:36.560 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --new-format /tmp/rbd_import_export_79662/img testimg 2024-09-21T09:51:36.825 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-21T09:51:37.063 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:37.093 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-09-21T09:51:37.094 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:37.097 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export testimg /tmp/rbd_import_export_79662/img2 2024-09-21T09:51:37.288 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:37.293 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export testimg - 2024-09-21T09:51:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:37 smithi032 ceph-mon[40061]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:37 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:37 smithi032 ceph-mon[29191]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:37 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:37.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:37 smithi045 ceph-mon[33087]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:51:37.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:37 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:37.897 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:37.901 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm testimg 2024-09-21T09:51:37.993 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-21T09:51:37.996 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp /tmp/rbd_import_export_79662/img /tmp/rbd_import_export_79662/img2 2024-09-21T09:51:38.626 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp /tmp/rbd_import_export_79662/img /tmp/rbd_import_export_79662/img3 2024-09-21T09:51:39.110 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/img2 /tmp/rbd_import_export_79662/img3 2024-09-21T09:51:39.282 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --new-format - testimg 2024-09-21T09:51:39.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:39 smithi045 ceph-mon[33087]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-09-21T09:51:39.672 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete...done. 2024-09-21T09:51:39.672 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:39.677 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export testimg /tmp/rbd_import_export_79662/img2 2024-09-21T09:51:39.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:39 smithi032 ceph-mon[29191]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-09-21T09:51:39.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:39 smithi032 ceph-mon[40061]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-09-21T09:51:39.867 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:39.872 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export testimg - 2024-09-21T09:51:40.514 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:40.518 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm testimg 2024-09-21T09:51:40.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:40 smithi045 ceph-mon[33087]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:40.616 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-21T09:51:40.619 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp /tmp/rbd_import_export_79662/img /tmp/rbd_import_export_79662/img2 2024-09-21T09:51:40.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:40 smithi032 ceph-mon[29191]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:40.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:40 smithi032 ceph-mon[40061]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:41.274 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp /tmp/rbd_import_export_79662/img /tmp/rbd_import_export_79662/img3 2024-09-21T09:51:41.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:41 smithi032 ceph-mon[40061]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-09-21T09:51:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:41 smithi032 ceph-mon[29191]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-09-21T09:51:41.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:41 smithi045 ceph-mon[33087]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-09-21T09:51:41.761 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/img /tmp/rbd_import_export_79662/img2 /tmp/rbd_import_export_79662/img3 2024-09-21T09:51:42.009 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd help export 2024-09-21T09:51:42.009 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep -q export-format 2024-09-21T09:51:42.032 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_79662/img bs=1k count=10 seek=100 2024-09-21T09:51:42.033 INFO:tasks.workunit.client.0.smithi032.stderr:10+0 records in 2024-09-21T09:51:42.034 INFO:tasks.workunit.client.0.smithi032.stderr:10+0 records out 2024-09-21T09:51:42.034 INFO:tasks.workunit.client.0.smithi032.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.6359e-05 s, 154 MB/s 2024-09-21T09:51:42.034 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --new-format /tmp/rbd_import_export_79662/img testimg 2024-09-21T09:51:42.096 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete...done. 2024-09-21T09:51:42.096 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:42.099 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap create testimg@snap 2024-09-21T09:51:42.988 INFO:tasks.workunit.client.0.smithi032.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-21T09:51:42.995 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd image-meta set testimg key1 value1 2024-09-21T09:51:43.052 INFO:tasks.workunit.client.0.smithi032.stderr:++ rbd image-meta list testimg 2024-09-21T09:51:43.099 INFO:tasks.workunit.client.0.smithi032.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2024-09-21T09:51:43.100 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-09-21T09:51:43.100 INFO:tasks.workunit.client.0.smithi032.stderr:Key Value 2024-09-21T09:51:43.100 INFO:tasks.workunit.client.0.smithi032.stderr:key1 value1' 2024-09-21T09:51:43.100 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79662/img_v2 2024-09-21T09:51:43.147 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:43.150 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79662/img_v2 testimg_import 2024-09-21T09:51:43.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:43 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:51:43] "GET /metrics HTTP/1.1" 200 33703 "" "Prometheus/2.43.0" 2024-09-21T09:51:44.001 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-21T09:51:44.004 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd info testimg_import 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout:rbd image 'testimg_import': 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: size 110 KiB in 1 objects 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: order 22 (4 MiB objects) 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: snapshot_count: 1 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: id: 3a383e8a1c5f 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: block_name_prefix: rbd_data.3a383e8a1c5f 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: format: 2 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: op_features: 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: flags: 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: create_timestamp: Sat Sep 21 09:51:43 2024 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: access_timestamp: Sat Sep 21 09:51:43 2024 2024-09-21T09:51:44.049 INFO:tasks.workunit.client.0.smithi032.stdout: modify_timestamp: Sat Sep 21 09:51:43 2024 2024-09-21T09:51:44.052 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd info testimg_import@snap 2024-09-21T09:51:44.094 INFO:tasks.workunit.client.0.smithi032.stdout:rbd image 'testimg_import': 2024-09-21T09:51:44.094 INFO:tasks.workunit.client.0.smithi032.stdout: size 110 KiB in 1 objects 2024-09-21T09:51:44.094 INFO:tasks.workunit.client.0.smithi032.stdout: order 22 (4 MiB objects) 2024-09-21T09:51:44.094 INFO:tasks.workunit.client.0.smithi032.stdout: snapshot_count: 1 2024-09-21T09:51:44.094 INFO:tasks.workunit.client.0.smithi032.stdout: id: 3a383e8a1c5f 2024-09-21T09:51:44.094 INFO:tasks.workunit.client.0.smithi032.stdout: block_name_prefix: rbd_data.3a383e8a1c5f 2024-09-21T09:51:44.094 INFO:tasks.workunit.client.0.smithi032.stdout: format: 2 2024-09-21T09:51:44.094 INFO:tasks.workunit.client.0.smithi032.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-09-21T09:51:44.095 INFO:tasks.workunit.client.0.smithi032.stdout: op_features: 2024-09-21T09:51:44.095 INFO:tasks.workunit.client.0.smithi032.stdout: flags: 2024-09-21T09:51:44.095 INFO:tasks.workunit.client.0.smithi032.stdout: create_timestamp: Sat Sep 21 09:51:43 2024 2024-09-21T09:51:44.095 INFO:tasks.workunit.client.0.smithi032.stdout: access_timestamp: Sat Sep 21 09:51:43 2024 2024-09-21T09:51:44.095 INFO:tasks.workunit.client.0.smithi032.stdout: modify_timestamp: Sat Sep 21 09:51:43 2024 2024-09-21T09:51:44.095 INFO:tasks.workunit.client.0.smithi032.stdout: protected: False 2024-09-21T09:51:44.097 INFO:tasks.workunit.client.0.smithi032.stderr:++ rbd image-meta list testimg_import 2024-09-21T09:51:44.146 INFO:tasks.workunit.client.0.smithi032.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2024-09-21T09:51:44.146 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr:Key Value 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr:key1 value1' 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' 'There is 1 metadatum on this image: 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr:Key Value 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr:key1 value1' = 'There is 1 metadatum on this image: 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr: 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr:Key Value 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr:key1 value1' ']' 2024-09-21T09:51:44.147 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export testimg_import /tmp/rbd_import_export_79662/img_import 2024-09-21T09:51:44.190 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 100% complete...done. 2024-09-21T09:51:44.194 INFO:tasks.workunit.client.0.smithi032.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79662/img /tmp/rbd_import_export_79662/img_import 2024-09-21T09:51:44.194 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp -l /tmp/rbd_import_export_79662/img /tmp/rbd_import_export_79662/img_import 2024-09-21T09:51:44.195 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/img --format %b 2024-09-21T09:51:44.196 INFO:tasks.workunit.client.0.smithi032.stderr:+ origsize=24 2024-09-21T09:51:44.196 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/img_import --format %b 2024-09-21T09:51:44.198 INFO:tasks.workunit.client.0.smithi032.stderr:+ exportsize=224 2024-09-21T09:51:44.198 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=200 2024-09-21T09:51:44.198 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=200 2024-09-21T09:51:44.198 INFO:tasks.workunit.client.0.smithi032.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-09-21T09:51:44.198 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_79662/img_snap 2024-09-21T09:51:44.242 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 100% complete...done. 2024-09-21T09:51:44.245 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_79662/img_snap_import 2024-09-21T09:51:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:43 smithi045 ceph-mon[33087]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:51:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:43 smithi045 ceph-mon[33087]: pgmap v96: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 95 KiB/s wr, 104 op/s 2024-09-21T09:51:44.289 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 100% complete...done. 2024-09-21T09:51:44.292 INFO:tasks.workunit.client.0.smithi032.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79662/img_snap /tmp/rbd_import_export_79662/img_snap_import 2024-09-21T09:51:44.293 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp -l /tmp/rbd_import_export_79662/img_snap /tmp/rbd_import_export_79662/img_snap_import 2024-09-21T09:51:44.294 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/img_snap --format %b 2024-09-21T09:51:44.294 INFO:tasks.workunit.client.0.smithi032.stderr:+ origsize=224 2024-09-21T09:51:44.295 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/img_snap_import --format %b 2024-09-21T09:51:44.296 INFO:tasks.workunit.client.0.smithi032.stderr:+ exportsize=224 2024-09-21T09:51:44.296 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:44.296 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:44.296 INFO:tasks.workunit.client.0.smithi032.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-21T09:51:44.296 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/img_v2 2024-09-21T09:51:44.297 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/img_import 2024-09-21T09:51:44.297 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/img_snap 2024-09-21T09:51:44.298 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/img_snap_import 2024-09-21T09:51:44.299 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap rm testimg_import@snap 2024-09-21T09:51:44.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:43 smithi032 ceph-mon[29191]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:51:44.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:43 smithi032 ceph-mon[29191]: pgmap v96: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 95 KiB/s wr, 104 op/s 2024-09-21T09:51:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:43 smithi032 ceph-mon[40061]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:51:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:43 smithi032 ceph-mon[40061]: pgmap v96: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 95 KiB/s wr, 104 op/s 2024-09-21T09:51:44.993 INFO:tasks.workunit.client.0.smithi032.stderr: Removing snap: 100% complete...done. 2024-09-21T09:51:45.002 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd remove testimg_import 2024-09-21T09:51:45.100 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:45.103 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap rm testimg@snap 2024-09-21T09:51:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:44 smithi045 ceph-mon[33087]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:51:45.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:44 smithi032 ceph-mon[40061]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:51:45.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:44 smithi032 ceph-mon[29191]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:51:45.998 INFO:tasks.workunit.client.0.smithi032.stderr: Removing snap: 100% complete...done. 2024-09-21T09:51:46.007 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm testimg 2024-09-21T09:51:46.105 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:46.108 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --order 20 /tmp/rbd_import_export_79662/img testimg 2024-09-21T09:51:46.172 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete...done. 2024-09-21T09:51:46.173 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --order is deprecated, use --object-size 2024-09-21T09:51:46.176 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79662/img_v2 2024-09-21T09:51:46.220 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:46.223 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79662/img_v2 testimg_import 2024-09-21T09:51:46.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:45 smithi045 ceph-mon[33087]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:51:46.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:45 smithi045 ceph-mon[33087]: pgmap v99: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 236 KiB/s rd, 83 KiB/s wr, 122 op/s 2024-09-21T09:51:46.290 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-21T09:51:46.294 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd info testimg_import 2024-09-21T09:51:46.294 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep order 2024-09-21T09:51:46.294 INFO:tasks.workunit.client.0.smithi032.stderr:+ awk '{print $2}' 2024-09-21T09:51:46.294 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep 20 2024-09-21T09:51:46.336 INFO:tasks.workunit.client.0.smithi032.stdout:20 2024-09-21T09:51:46.336 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/img_v2 2024-09-21T09:51:46.337 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd remove testimg_import 2024-09-21T09:51:46.434 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:46.438 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd remove testimg 2024-09-21T09:51:46.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:45 smithi032 ceph-mon[40061]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:51:46.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:45 smithi032 ceph-mon[40061]: pgmap v99: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 236 KiB/s rd, 83 KiB/s wr, 122 op/s 2024-09-21T09:51:46.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:45 smithi032 ceph-mon[29191]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:51:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:45 smithi032 ceph-mon[29191]: pgmap v99: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 236 KiB/s rd, 83 KiB/s wr, 122 op/s 2024-09-21T09:51:46.530 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:46.533 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_79662/img testimg 2024-09-21T09:51:46.587 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete...done. 2024-09-21T09:51:46.591 INFO:tasks.workunit.client.0.smithi032.stderr:++ rbd info testimg 2024-09-21T09:51:46.591 INFO:tasks.workunit.client.0.smithi032.stderr:++ grep features 2024-09-21T09:51:46.637 INFO:tasks.workunit.client.0.smithi032.stderr:+ FEATURES_BEFORE=' features: layering 2024-09-21T09:51:46.637 INFO:tasks.workunit.client.0.smithi032.stderr: op_features: ' 2024-09-21T09:51:46.637 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79662/img_v2 2024-09-21T09:51:46.683 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:46.686 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79662/img_v2 testimg_import 2024-09-21T09:51:46.744 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-21T09:51:46.748 INFO:tasks.workunit.client.0.smithi032.stderr:++ rbd info testimg_import 2024-09-21T09:51:46.748 INFO:tasks.workunit.client.0.smithi032.stderr:++ grep features 2024-09-21T09:51:46.793 INFO:tasks.workunit.client.0.smithi032.stderr:+ FEATURES_AFTER=' features: layering 2024-09-21T09:51:46.793 INFO:tasks.workunit.client.0.smithi032.stderr: op_features: ' 2024-09-21T09:51:46.793 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' ' features: layering 2024-09-21T09:51:46.793 INFO:tasks.workunit.client.0.smithi032.stderr: op_features: ' '!=' ' features: layering 2024-09-21T09:51:46.793 INFO:tasks.workunit.client.0.smithi032.stderr: op_features: ' ']' 2024-09-21T09:51:46.793 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/img_v2 2024-09-21T09:51:46.794 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd remove testimg_import 2024-09-21T09:51:46.868 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:46.871 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd remove testimg 2024-09-21T09:51:46.948 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:46.952 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_79662/img testimg 2024-09-21T09:51:47.020 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete...done. 2024-09-21T09:51:47.024 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79662/img_v2 2024-09-21T09:51:47.076 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:47.079 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79662/img_v2 testimg_import 2024-09-21T09:51:47.153 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-21T09:51:47.157 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd info testimg_import 2024-09-21T09:51:47.157 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep 'stripe unit' 2024-09-21T09:51:47.158 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep -Ei '(4 KiB|4096)' 2024-09-21T09:51:47.204 INFO:tasks.workunit.client.0.smithi032.stdout: stripe unit: 4 KiB 2024-09-21T09:51:47.205 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd info testimg_import 2024-09-21T09:51:47.205 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep 'stripe count' 2024-09-21T09:51:47.205 INFO:tasks.workunit.client.0.smithi032.stderr:+ awk '{print $3}' 2024-09-21T09:51:47.205 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep 1000 2024-09-21T09:51:47.248 INFO:tasks.workunit.client.0.smithi032.stdout:1000 2024-09-21T09:51:47.249 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/img_v2 2024-09-21T09:51:47.249 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd remove testimg_import 2024-09-21T09:51:47.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:46 smithi045 ceph-mon[33087]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:51:47.347 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-21T09:51:47.350 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd remove testimg 2024-09-21T09:51:47.442 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-21T09:51:47.445 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_79662/img testimg 2024-09-21T09:51:47.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:46 smithi032 ceph-mon[40061]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:51:47.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:46 smithi032 ceph-mon[29191]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:51:47.506 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete...done. 2024-09-21T09:51:47.509 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap create testimg@snap1 2024-09-21T09:51:48.008 INFO:tasks.workunit.client.0.smithi032.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-21T09:51:48.015 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap create testimg@snap2 2024-09-21T09:51:48.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:47 smithi045 ceph-mon[33087]: pgmap v101: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 338 KiB/s rd, 118 KiB/s wr, 175 op/s 2024-09-21T09:51:48.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:47 smithi032 ceph-mon[29191]: pgmap v101: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 338 KiB/s rd, 118 KiB/s wr, 175 op/s 2024-09-21T09:51:48.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:47 smithi032 ceph-mon[40061]: pgmap v101: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 338 KiB/s rd, 118 KiB/s wr, 175 op/s 2024-09-21T09:51:49.010 INFO:tasks.workunit.client.0.smithi032.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-21T09:51:49.017 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap protect testimg@snap2 2024-09-21T09:51:49.070 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_79662/snap_protect 2024-09-21T09:51:49.117 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:49.120 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_79662/snap_protect testimg_import 2024-09-21T09:51:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:49 smithi045 ceph-mon[33087]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:51:49.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:49 smithi032 ceph-mon[29191]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:51:49.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:49 smithi032 ceph-mon[40061]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:51:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:50 smithi045 ceph-mon[33087]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:51:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:50 smithi045 ceph-mon[33087]: pgmap v104: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 448 KiB/s rd, 50 KiB/s wr, 590 op/s 2024-09-21T09:51:50.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:50 smithi032 ceph-mon[29191]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:51:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:50 smithi032 ceph-mon[29191]: pgmap v104: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 448 KiB/s rd, 50 KiB/s wr, 590 op/s 2024-09-21T09:51:50.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:50 smithi032 ceph-mon[40061]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:51:50.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:50 smithi032 ceph-mon[40061]: pgmap v104: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 448 KiB/s rd, 50 KiB/s wr, 590 op/s 2024-09-21T09:51:51.032 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-21T09:51:51.035 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd info testimg_import@snap1 2024-09-21T09:51:51.035 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep 'protected: False' 2024-09-21T09:51:51.082 INFO:tasks.workunit.client.0.smithi032.stdout: protected: False 2024-09-21T09:51:51.082 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd info testimg_import@snap2 2024-09-21T09:51:51.082 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep 'protected: True' 2024-09-21T09:51:51.127 INFO:tasks.workunit.client.0.smithi032.stdout: protected: True 2024-09-21T09:51:51.127 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/snap_protect 2024-09-21T09:51:51.128 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap unprotect testimg@snap2 2024-09-21T09:51:51.180 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap unprotect testimg_import@snap2 2024-09-21T09:51:51.231 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap purge testimg 2024-09-21T09:51:51.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:51 smithi045 ceph-mon[33087]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:51:51.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:51 smithi032 ceph-mon[29191]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:51:51.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:51 smithi032 ceph-mon[40061]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:51:52.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:52 smithi045 ceph-mon[33087]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:51:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:52 smithi045 ceph-mon[33087]: pgmap v107: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 468 KiB/s rd, 52 KiB/s wr, 618 op/s 2024-09-21T09:51:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:52 smithi045 ceph-mon[33087]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:51:52.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:52 smithi032 ceph-mon[29191]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:51:52.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:52 smithi032 ceph-mon[29191]: pgmap v107: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 468 KiB/s rd, 52 KiB/s wr, 618 op/s 2024-09-21T09:51:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:52 smithi032 ceph-mon[29191]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:51:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:52 smithi032 ceph-mon[40061]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:51:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:52 smithi032 ceph-mon[40061]: pgmap v107: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 468 KiB/s rd, 52 KiB/s wr, 618 op/s 2024-09-21T09:51:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:52 smithi032 ceph-mon[40061]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:51:52.984 INFO:tasks.workunit.client.0.smithi032.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-21T09:51:52.993 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd snap purge testimg_import 2024-09-21T09:51:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:53 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:53 smithi045 ceph-mon[33087]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:51:53.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:53 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:53.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:53 smithi032 ceph-mon[29191]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:51:53.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:53 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:53.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:53 smithi032 ceph-mon[40061]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:51:53.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:51:53 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:51:53] "GET /metrics HTTP/1.1" 200 33721 "" "Prometheus/2.43.0" 2024-09-21T09:51:54.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:54 smithi045 ceph-mon[33087]: pgmap v110: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-09-21T09:51:54.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:54 smithi032 ceph-mon[29191]: pgmap v110: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-09-21T09:51:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:54 smithi032 ceph-mon[40061]: pgmap v110: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-09-21T09:51:55.032 INFO:tasks.workunit.client.0.smithi032.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-21T09:51:55.039 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd remove testimg 2024-09-21T09:51:55.138 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:55.141 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd remove testimg_import 2024-09-21T09:51:55.236 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:55.239 INFO:tasks.workunit.client.0.smithi032.stderr:+ tiered=0 2024-09-21T09:51:55.240 INFO:tasks.workunit.client.0.smithi032.stderr:+ ceph osd dump 2024-09-21T09:51:55.240 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep '^pool' 2024-09-21T09:51:55.240 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep ''\''rbd'\''' 2024-09-21T09:51:55.240 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep tier 2024-09-21T09:51:55.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:55 smithi045 ceph-mon[33087]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:51:55.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:55 smithi032 ceph-mon[40061]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:51:55.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:55 smithi032 ceph-mon[29191]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:51:55.671 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_79662/sparse1 2024-09-21T09:51:55.675 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records in 2024-09-21T09:51:55.676 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records out 2024-09-21T09:51:55.676 INFO:tasks.workunit.client.0.smithi032.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00317129 s, 331 MB/s 2024-09-21T09:51:55.676 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_79662/sparse2 2024-09-21T09:51:55.680 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records in 2024-09-21T09:51:55.680 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records out 2024-09-21T09:51:55.680 INFO:tasks.workunit.client.0.smithi032.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0031189 s, 336 MB/s 2024-09-21T09:51:55.680 INFO:tasks.workunit.client.0.smithi032.stderr:+ truncate /tmp/rbd_import_export_79662/sparse2 -s 2M 2024-09-21T09:51:55.699 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm sparse1 2024-09-21T09:51:55.744 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 0% complete...failed. 2024-09-21T09:51:55.744 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: delete error: (2) No such file or directory 2024-09-21T09:51:55.747 INFO:tasks.workunit.client.0.smithi032.stderr:+ true 2024-09-21T09:51:55.748 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_79662/sparse1 2024-09-21T09:51:55.818 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-09-21T09:51:55.819 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:55.819 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --order is deprecated, use --object-size 2024-09-21T09:51:55.822 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd ls -l 2024-09-21T09:51:55.822 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep sparse1 2024-09-21T09:51:55.822 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-09-21T09:51:55.867 INFO:tasks.workunit.client.0.smithi032.stdout:sparse1 2 MiB 2 2024-09-21T09:51:55.868 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects sparse1 2024-09-21T09:51:55.868 INFO:tasks.workunit.client.0.smithi032.stderr:++ image=sparse1 2024-09-21T09:51:55.868 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rbd info sparse1 2024-09-21T09:51:55.868 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep block_name_prefix 2024-09-21T09:51:55.868 INFO:tasks.workunit.client.0.smithi032.stderr:+++ awk '{print $NF;}' 2024-09-21T09:51:55.912 INFO:tasks.workunit.client.0.smithi032.stderr:++ prefix=rbd_data.61f44e11d77f 2024-09-21T09:51:55.913 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep rbd_data.61f44e11d77f 2024-09-21T09:51:55.913 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sed -e 's/rbd_data.61f44e11d77f\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-21T09:51:55.913 INFO:tasks.workunit.client.0.smithi032.stderr:++++ get_image_data_pool sparse1 2024-09-21T09:51:55.913 INFO:tasks.workunit.client.0.smithi032.stderr:++++ image=sparse1 2024-09-21T09:51:55.913 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sort -u 2024-09-21T09:51:55.914 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ rbd info sparse1 2024-09-21T09:51:55.914 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ grep 'data_pool: ' 2024-09-21T09:51:55.914 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ awk -F: '{ print $NF }' 2024-09-21T09:51:55.958 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool= 2024-09-21T09:51:55.958 INFO:tasks.workunit.client.0.smithi032.stderr:++++ '[' -z ']' 2024-09-21T09:51:55.958 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool=rbd 2024-09-21T09:51:55.958 INFO:tasks.workunit.client.0.smithi032.stderr:++++ echo rbd 2024-09-21T09:51:55.958 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rados ls -p rbd 2024-09-21T09:51:56.000 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects=1 2024-09-21T09:51:56.000 INFO:tasks.workunit.client.0.smithi032.stderr:++ echo 1 2024-09-21T09:51:56.000 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-09-21T09:51:56.000 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export sparse1 /tmp/rbd_import_export_79662/sparse1.out 2024-09-21T09:51:56.047 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:56.050 INFO:tasks.workunit.client.0.smithi032.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79662/sparse1 /tmp/rbd_import_export_79662/sparse1.out 2024-09-21T09:51:56.051 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp -l /tmp/rbd_import_export_79662/sparse1 /tmp/rbd_import_export_79662/sparse1.out 2024-09-21T09:51:56.052 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/sparse1 --format %b 2024-09-21T09:51:56.054 INFO:tasks.workunit.client.0.smithi032.stderr:+ origsize=2048 2024-09-21T09:51:56.054 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/sparse1.out --format %b 2024-09-21T09:51:56.055 INFO:tasks.workunit.client.0.smithi032.stderr:+ exportsize=2048 2024-09-21T09:51:56.055 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:56.055 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:56.055 INFO:tasks.workunit.client.0.smithi032.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-21T09:51:56.055 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/sparse1.out 2024-09-21T09:51:56.056 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm sparse1 2024-09-21T09:51:56.151 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-09-21T09:51:56.155 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm sparse2 2024-09-21T09:51:56.199 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 0% complete...failed. 2024-09-21T09:51:56.199 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: delete error: (2) No such file or directory 2024-09-21T09:51:56.203 INFO:tasks.workunit.client.0.smithi032.stderr:+ true 2024-09-21T09:51:56.203 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_79662/sparse2 2024-09-21T09:51:56.271 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-09-21T09:51:56.272 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:56.272 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --order is deprecated, use --object-size 2024-09-21T09:51:56.275 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd ls -l 2024-09-21T09:51:56.275 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep sparse2 2024-09-21T09:51:56.275 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-09-21T09:51:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:56 smithi045 ceph-mon[33087]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:51:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:56 smithi045 ceph-mon[33087]: pgmap v113: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-09-21T09:51:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:56 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/2723196427' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-21T09:51:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:56 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/2723196427' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-21T09:51:56.319 INFO:tasks.workunit.client.0.smithi032.stdout:sparse2 2 MiB 2 2024-09-21T09:51:56.319 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects sparse2 2024-09-21T09:51:56.320 INFO:tasks.workunit.client.0.smithi032.stderr:++ image=sparse2 2024-09-21T09:51:56.320 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rbd info sparse2 2024-09-21T09:51:56.320 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep block_name_prefix 2024-09-21T09:51:56.320 INFO:tasks.workunit.client.0.smithi032.stderr:+++ awk '{print $NF;}' 2024-09-21T09:51:56.364 INFO:tasks.workunit.client.0.smithi032.stderr:++ prefix=rbd_data.3b859dd2c7be 2024-09-21T09:51:56.365 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep rbd_data.3b859dd2c7be 2024-09-21T09:51:56.365 INFO:tasks.workunit.client.0.smithi032.stderr:++++ get_image_data_pool sparse2 2024-09-21T09:51:56.365 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sed -e 's/rbd_data.3b859dd2c7be\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-21T09:51:56.366 INFO:tasks.workunit.client.0.smithi032.stderr:++++ image=sparse2 2024-09-21T09:51:56.366 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sort -u 2024-09-21T09:51:56.366 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ rbd info sparse2 2024-09-21T09:51:56.366 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ grep 'data_pool: ' 2024-09-21T09:51:56.366 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ awk -F: '{ print $NF }' 2024-09-21T09:51:56.411 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool= 2024-09-21T09:51:56.411 INFO:tasks.workunit.client.0.smithi032.stderr:++++ '[' -z ']' 2024-09-21T09:51:56.411 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool=rbd 2024-09-21T09:51:56.411 INFO:tasks.workunit.client.0.smithi032.stderr:++++ echo rbd 2024-09-21T09:51:56.411 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rados ls -p rbd 2024-09-21T09:51:56.451 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects=0 2024-09-21T09:51:56.451 INFO:tasks.workunit.client.0.smithi032.stderr:++ echo 0 2024-09-21T09:51:56.451 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-09-21T09:51:56.451 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export sparse2 /tmp/rbd_import_export_79662/sparse2.out 2024-09-21T09:51:56.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:56 smithi032 ceph-mon[40061]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:51:56.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:56 smithi032 ceph-mon[40061]: pgmap v113: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-09-21T09:51:56.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:56 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/2723196427' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-21T09:51:56.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:56 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/2723196427' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-21T09:51:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:56 smithi032 ceph-mon[29191]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:51:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:56 smithi032 ceph-mon[29191]: pgmap v113: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-09-21T09:51:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:56 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2723196427' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-21T09:51:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:56 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2723196427' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-21T09:51:56.496 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:56.500 INFO:tasks.workunit.client.0.smithi032.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79662/sparse2 /tmp/rbd_import_export_79662/sparse2.out 2024-09-21T09:51:56.500 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp -l /tmp/rbd_import_export_79662/sparse2 /tmp/rbd_import_export_79662/sparse2.out 2024-09-21T09:51:56.502 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/sparse2 --format %b 2024-09-21T09:51:56.503 INFO:tasks.workunit.client.0.smithi032.stderr:+ origsize=2048 2024-09-21T09:51:56.504 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/sparse2.out --format %b 2024-09-21T09:51:56.505 INFO:tasks.workunit.client.0.smithi032.stderr:+ exportsize=2048 2024-09-21T09:51:56.505 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:56.505 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:56.505 INFO:tasks.workunit.client.0.smithi032.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-21T09:51:56.505 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/sparse2.out 2024-09-21T09:51:56.505 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm sparse2 2024-09-21T09:51:56.604 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-09-21T09:51:56.607 INFO:tasks.workunit.client.0.smithi032.stderr:+ truncate /tmp/rbd_import_export_79662/sparse1 -s 10M 2024-09-21T09:51:56.608 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --new-format --order 20 - sparse1 2024-09-21T09:51:56.702 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete...done. 2024-09-21T09:51:56.703 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:56.703 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --order is deprecated, use --object-size 2024-09-21T09:51:56.706 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd ls -l 2024-09-21T09:51:56.706 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep sparse1 2024-09-21T09:51:56.706 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep -Ei '(10 MiB|10240k)' 2024-09-21T09:51:56.750 INFO:tasks.workunit.client.0.smithi032.stdout:sparse1 10 MiB 2 2024-09-21T09:51:56.750 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects sparse1 2024-09-21T09:51:56.750 INFO:tasks.workunit.client.0.smithi032.stderr:++ image=sparse1 2024-09-21T09:51:56.751 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rbd info sparse1 2024-09-21T09:51:56.751 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep block_name_prefix 2024-09-21T09:51:56.751 INFO:tasks.workunit.client.0.smithi032.stderr:+++ awk '{print $NF;}' 2024-09-21T09:51:56.796 INFO:tasks.workunit.client.0.smithi032.stderr:++ prefix=rbd_data.6254ee9bf7b0 2024-09-21T09:51:56.796 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep rbd_data.6254ee9bf7b0 2024-09-21T09:51:56.797 INFO:tasks.workunit.client.0.smithi032.stderr:++++ get_image_data_pool sparse1 2024-09-21T09:51:56.797 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sed -e 's/rbd_data.6254ee9bf7b0\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-21T09:51:56.797 INFO:tasks.workunit.client.0.smithi032.stderr:++++ image=sparse1 2024-09-21T09:51:56.797 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sort -u 2024-09-21T09:51:56.797 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ rbd info sparse1 2024-09-21T09:51:56.797 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ grep 'data_pool: ' 2024-09-21T09:51:56.798 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ awk -F: '{ print $NF }' 2024-09-21T09:51:56.842 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool= 2024-09-21T09:51:56.842 INFO:tasks.workunit.client.0.smithi032.stderr:++++ '[' -z ']' 2024-09-21T09:51:56.842 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool=rbd 2024-09-21T09:51:56.842 INFO:tasks.workunit.client.0.smithi032.stderr:++++ echo rbd 2024-09-21T09:51:56.842 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rados ls -p rbd 2024-09-21T09:51:56.882 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects=1 2024-09-21T09:51:56.882 INFO:tasks.workunit.client.0.smithi032.stderr:++ echo 1 2024-09-21T09:51:56.883 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-09-21T09:51:56.883 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export sparse1 /tmp/rbd_import_export_79662/sparse1.out 2024-09-21T09:51:56.926 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:56.930 INFO:tasks.workunit.client.0.smithi032.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79662/sparse1 /tmp/rbd_import_export_79662/sparse1.out 2024-09-21T09:51:56.930 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp -l /tmp/rbd_import_export_79662/sparse1 /tmp/rbd_import_export_79662/sparse1.out 2024-09-21T09:51:56.938 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/sparse1 --format %b 2024-09-21T09:51:56.939 INFO:tasks.workunit.client.0.smithi032.stderr:+ origsize=2048 2024-09-21T09:51:56.939 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/sparse1.out --format %b 2024-09-21T09:51:56.940 INFO:tasks.workunit.client.0.smithi032.stderr:+ exportsize=2048 2024-09-21T09:51:56.940 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:56.940 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:56.940 INFO:tasks.workunit.client.0.smithi032.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-21T09:51:56.940 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/sparse1.out 2024-09-21T09:51:56.941 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm sparse1 2024-09-21T09:51:57.040 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-09-21T09:51:57.044 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_79662/sparse2 oflag=append conv=notrunc 2024-09-21T09:51:57.051 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records in 2024-09-21T09:51:57.051 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records out 2024-09-21T09:51:57.051 INFO:tasks.workunit.client.0.smithi032.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00605826 s, 346 MB/s 2024-09-21T09:51:57.051 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --new-format --order 20 - sparse2 2024-09-21T09:51:57.136 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete...done. 2024-09-21T09:51:57.137 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:57.137 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --order is deprecated, use --object-size 2024-09-21T09:51:57.140 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd ls -l 2024-09-21T09:51:57.140 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep sparse2 2024-09-21T09:51:57.140 INFO:tasks.workunit.client.0.smithi032.stderr:+ grep -Ei '(4 MiB|4096k)' 2024-09-21T09:51:57.185 INFO:tasks.workunit.client.0.smithi032.stdout:sparse2 4 MiB 2 2024-09-21T09:51:57.185 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects sparse2 2024-09-21T09:51:57.185 INFO:tasks.workunit.client.0.smithi032.stderr:++ image=sparse2 2024-09-21T09:51:57.186 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rbd info sparse2 2024-09-21T09:51:57.186 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep block_name_prefix 2024-09-21T09:51:57.186 INFO:tasks.workunit.client.0.smithi032.stderr:+++ awk '{print $NF;}' 2024-09-21T09:51:57.229 INFO:tasks.workunit.client.0.smithi032.stderr:++ prefix=rbd_data.62816eed71b6 2024-09-21T09:51:57.230 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep rbd_data.62816eed71b6 2024-09-21T09:51:57.230 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sed -e 's/rbd_data.62816eed71b6\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-21T09:51:57.230 INFO:tasks.workunit.client.0.smithi032.stderr:++++ get_image_data_pool sparse2 2024-09-21T09:51:57.230 INFO:tasks.workunit.client.0.smithi032.stderr:++++ image=sparse2 2024-09-21T09:51:57.231 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sort -u 2024-09-21T09:51:57.231 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ rbd info sparse2 2024-09-21T09:51:57.231 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ grep 'data_pool: ' 2024-09-21T09:51:57.231 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ awk -F: '{ print $NF }' 2024-09-21T09:51:57.275 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool= 2024-09-21T09:51:57.275 INFO:tasks.workunit.client.0.smithi032.stderr:++++ '[' -z ']' 2024-09-21T09:51:57.275 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool=rbd 2024-09-21T09:51:57.275 INFO:tasks.workunit.client.0.smithi032.stderr:++++ echo rbd 2024-09-21T09:51:57.275 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rados ls -p rbd 2024-09-21T09:51:57.316 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects='0 2024-09-21T09:51:57.316 INFO:tasks.workunit.client.0.smithi032.stderr:2 2024-09-21T09:51:57.316 INFO:tasks.workunit.client.0.smithi032.stderr:3' 2024-09-21T09:51:57.316 INFO:tasks.workunit.client.0.smithi032.stderr:++ echo 0 2 3 2024-09-21T09:51:57.316 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2024-09-21T09:51:57.316 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export sparse2 /tmp/rbd_import_export_79662/sparse2.out 2024-09-21T09:51:57.362 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-09-21T09:51:57.366 INFO:tasks.workunit.client.0.smithi032.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_79662/sparse2 /tmp/rbd_import_export_79662/sparse2.out 2024-09-21T09:51:57.366 INFO:tasks.workunit.client.0.smithi032.stderr:+ cmp -l /tmp/rbd_import_export_79662/sparse2 /tmp/rbd_import_export_79662/sparse2.out 2024-09-21T09:51:57.369 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/sparse2 --format %b 2024-09-21T09:51:57.370 INFO:tasks.workunit.client.0.smithi032.stderr:+ origsize=6144 2024-09-21T09:51:57.370 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/sparse2.out --format %b 2024-09-21T09:51:57.372 INFO:tasks.workunit.client.0.smithi032.stderr:+ exportsize=6144 2024-09-21T09:51:57.372 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:57.372 INFO:tasks.workunit.client.0.smithi032.stderr:+ difference=0 2024-09-21T09:51:57.372 INFO:tasks.workunit.client.0.smithi032.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-21T09:51:57.372 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/sparse2.out 2024-09-21T09:51:57.372 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm sparse2 2024-09-21T09:51:57.464 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-09-21T09:51:57.468 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2024-09-21T09:51:57.468 INFO:tasks.workunit.client.0.smithi032.stdout:partially-sparse file imports to partially-sparse image 2024-09-21T09:51:57.468 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_79662/sparse1 sparse 2024-09-21T09:51:57.533 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-09-21T09:51:57.534 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:57.535 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --order is deprecated, use --object-size 2024-09-21T09:51:57.537 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects sparse 2024-09-21T09:51:57.538 INFO:tasks.workunit.client.0.smithi032.stderr:++ image=sparse 2024-09-21T09:51:57.538 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rbd info sparse 2024-09-21T09:51:57.538 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep block_name_prefix 2024-09-21T09:51:57.538 INFO:tasks.workunit.client.0.smithi032.stderr:+++ awk '{print $NF;}' 2024-09-21T09:51:57.582 INFO:tasks.workunit.client.0.smithi032.stderr:++ prefix=rbd_data.3c0960d6c50c 2024-09-21T09:51:57.583 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep rbd_data.3c0960d6c50c 2024-09-21T09:51:57.583 INFO:tasks.workunit.client.0.smithi032.stderr:++++ get_image_data_pool sparse 2024-09-21T09:51:57.583 INFO:tasks.workunit.client.0.smithi032.stderr:++++ image=sparse 2024-09-21T09:51:57.584 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sed -e 's/rbd_data.3c0960d6c50c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-21T09:51:57.584 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sort -u 2024-09-21T09:51:57.584 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ rbd info sparse 2024-09-21T09:51:57.584 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ grep 'data_pool: ' 2024-09-21T09:51:57.584 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ awk -F: '{ print $NF }' 2024-09-21T09:51:57.630 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool= 2024-09-21T09:51:57.630 INFO:tasks.workunit.client.0.smithi032.stderr:++++ '[' -z ']' 2024-09-21T09:51:57.630 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool=rbd 2024-09-21T09:51:57.630 INFO:tasks.workunit.client.0.smithi032.stderr:++++ echo rbd 2024-09-21T09:51:57.630 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rados ls -p rbd 2024-09-21T09:51:57.669 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects=1 2024-09-21T09:51:57.669 INFO:tasks.workunit.client.0.smithi032.stderr:++ echo 1 2024-09-21T09:51:57.669 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-09-21T09:51:57.670 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm sparse 2024-09-21T09:51:57.757 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-09-21T09:51:57.761 INFO:tasks.workunit.client.0.smithi032.stdout:zeros import through stdin to sparse image 2024-09-21T09:51:57.761 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo 'zeros import through stdin to sparse image' 2024-09-21T09:51:57.761 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/dev/zero bs=1M count=4 2024-09-21T09:51:57.761 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd import --new-format - sparse 2024-09-21T09:51:57.815 INFO:tasks.workunit.client.0.smithi032.stderr:4+0 records in 2024-09-21T09:51:57.816 INFO:tasks.workunit.client.0.smithi032.stderr:4+0 records out 2024-09-21T09:51:57.816 INFO:tasks.workunit.client.0.smithi032.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0535984 s, 78.3 MB/s 2024-09-21T09:51:57.826 INFO:tasks.workunit.client.0.smithi032.stderr: Importing image: 100% complete...done. 2024-09-21T09:51:57.827 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:57.831 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects sparse 2024-09-21T09:51:57.831 INFO:tasks.workunit.client.0.smithi032.stderr:++ image=sparse 2024-09-21T09:51:57.831 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rbd info sparse 2024-09-21T09:51:57.832 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep block_name_prefix 2024-09-21T09:51:57.832 INFO:tasks.workunit.client.0.smithi032.stderr:+++ awk '{print $NF;}' 2024-09-21T09:51:57.877 INFO:tasks.workunit.client.0.smithi032.stderr:++ prefix=rbd_data.3c2a7e7144b4 2024-09-21T09:51:57.878 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep rbd_data.3c2a7e7144b4 2024-09-21T09:51:57.878 INFO:tasks.workunit.client.0.smithi032.stderr:++++ get_image_data_pool sparse 2024-09-21T09:51:57.878 INFO:tasks.workunit.client.0.smithi032.stderr:++++ image=sparse 2024-09-21T09:51:57.878 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sed -e 's/rbd_data.3c2a7e7144b4\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-21T09:51:57.878 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sort -u 2024-09-21T09:51:57.878 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ rbd info sparse 2024-09-21T09:51:57.879 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ grep 'data_pool: ' 2024-09-21T09:51:57.879 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ awk -F: '{ print $NF }' 2024-09-21T09:51:57.922 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool= 2024-09-21T09:51:57.922 INFO:tasks.workunit.client.0.smithi032.stderr:++++ '[' -z ']' 2024-09-21T09:51:57.922 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool=rbd 2024-09-21T09:51:57.922 INFO:tasks.workunit.client.0.smithi032.stderr:++++ echo rbd 2024-09-21T09:51:57.922 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rados ls -p rbd 2024-09-21T09:51:57.962 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects= 2024-09-21T09:51:57.963 INFO:tasks.workunit.client.0.smithi032.stderr:++ echo 2024-09-21T09:51:57.963 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2024-09-21T09:51:57.963 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm sparse 2024-09-21T09:51:58.053 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:58.056 INFO:tasks.workunit.client.0.smithi032.stdout:zeros export to sparse file 2024-09-21T09:51:58.056 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo 'zeros export to sparse file' 2024-09-21T09:51:58.056 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd create --new-format sparse --size 4 2024-09-21T09:51:58.104 INFO:tasks.workunit.client.0.smithi032.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-21T09:51:58.107 INFO:tasks.workunit.client.0.smithi032.stderr:++ rbd info sparse 2024-09-21T09:51:58.107 INFO:tasks.workunit.client.0.smithi032.stderr:++ grep block_name_prefix 2024-09-21T09:51:58.108 INFO:tasks.workunit.client.0.smithi032.stderr:++ awk '{print $NF;}' 2024-09-21T09:51:58.152 INFO:tasks.workunit.client.0.smithi032.stderr:+ prefix=rbd_data.3c4b886c26c8 2024-09-21T09:51:58.153 INFO:tasks.workunit.client.0.smithi032.stderr:+ dd if=/dev/zero bs=4M count=1 2024-09-21T09:51:58.153 INFO:tasks.workunit.client.0.smithi032.stderr:++ get_image_data_pool sparse 2024-09-21T09:51:58.153 INFO:tasks.workunit.client.0.smithi032.stderr:++ image=sparse 2024-09-21T09:51:58.153 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rbd info sparse 2024-09-21T09:51:58.154 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep 'data_pool: ' 2024-09-21T09:51:58.154 INFO:tasks.workunit.client.0.smithi032.stderr:+++ awk -F: '{ print $NF }' 2024-09-21T09:51:58.195 INFO:tasks.workunit.client.0.smithi032.stderr:++ data_pool= 2024-09-21T09:51:58.196 INFO:tasks.workunit.client.0.smithi032.stderr:++ '[' -z ']' 2024-09-21T09:51:58.196 INFO:tasks.workunit.client.0.smithi032.stderr:++ data_pool=rbd 2024-09-21T09:51:58.196 INFO:tasks.workunit.client.0.smithi032.stderr:++ echo rbd 2024-09-21T09:51:58.196 INFO:tasks.workunit.client.0.smithi032.stderr:+ rados -p rbd put rbd_data.3c4b886c26c8.000000000000 - 2024-09-21T09:51:58.232 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records in 2024-09-21T09:51:58.232 INFO:tasks.workunit.client.0.smithi032.stderr:1+0 records out 2024-09-21T09:51:58.232 INFO:tasks.workunit.client.0.smithi032.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0801611 s, 52.3 MB/s 2024-09-21T09:51:58.255 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects sparse 2024-09-21T09:51:58.255 INFO:tasks.workunit.client.0.smithi032.stderr:++ image=sparse 2024-09-21T09:51:58.255 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rbd info sparse 2024-09-21T09:51:58.255 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep block_name_prefix 2024-09-21T09:51:58.256 INFO:tasks.workunit.client.0.smithi032.stderr:+++ awk '{print $NF;}' 2024-09-21T09:51:58.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:57 smithi032 ceph-mon[40061]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:51:58.258 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:57 smithi032 ceph-mon[40061]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 11 KiB/s wr, 163 op/s 2024-09-21T09:51:58.258 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:57 smithi032 ceph-mon[29191]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:51:58.258 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:57 smithi032 ceph-mon[29191]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 11 KiB/s wr, 163 op/s 2024-09-21T09:51:58.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:57 smithi045 ceph-mon[33087]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:51:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:57 smithi045 ceph-mon[33087]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 11 KiB/s wr, 163 op/s 2024-09-21T09:51:58.304 INFO:tasks.workunit.client.0.smithi032.stderr:++ prefix=rbd_data.3c4b886c26c8 2024-09-21T09:51:58.306 INFO:tasks.workunit.client.0.smithi032.stderr:+++ grep rbd_data.3c4b886c26c8 2024-09-21T09:51:58.306 INFO:tasks.workunit.client.0.smithi032.stderr:++++ get_image_data_pool sparse 2024-09-21T09:51:58.306 INFO:tasks.workunit.client.0.smithi032.stderr:++++ image=sparse 2024-09-21T09:51:58.306 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sed -e 's/rbd_data.3c4b886c26c8\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-21T09:51:58.307 INFO:tasks.workunit.client.0.smithi032.stderr:+++ sort -u 2024-09-21T09:51:58.307 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ rbd info sparse 2024-09-21T09:51:58.307 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ grep 'data_pool: ' 2024-09-21T09:51:58.307 INFO:tasks.workunit.client.0.smithi032.stderr:+++++ awk -F: '{ print $NF }' 2024-09-21T09:51:58.354 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool= 2024-09-21T09:51:58.354 INFO:tasks.workunit.client.0.smithi032.stderr:++++ '[' -z ']' 2024-09-21T09:51:58.354 INFO:tasks.workunit.client.0.smithi032.stderr:++++ data_pool=rbd 2024-09-21T09:51:58.354 INFO:tasks.workunit.client.0.smithi032.stderr:++++ echo rbd 2024-09-21T09:51:58.354 INFO:tasks.workunit.client.0.smithi032.stderr:+++ rados ls -p rbd 2024-09-21T09:51:58.394 INFO:tasks.workunit.client.0.smithi032.stderr:++ objects=0 2024-09-21T09:51:58.395 INFO:tasks.workunit.client.0.smithi032.stderr:++ echo 0 2024-09-21T09:51:58.395 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-09-21T09:51:58.395 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/sparse 2024-09-21T09:51:58.396 INFO:tasks.workunit.client.0.smithi032.stderr:rm: cannot remove '/tmp/rbd_import_export_79662/sparse': No such file or directory 2024-09-21T09:51:58.396 INFO:tasks.workunit.client.0.smithi032.stderr:+ true 2024-09-21T09:51:58.396 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd export sparse /tmp/rbd_import_export_79662/sparse 2024-09-21T09:51:58.440 INFO:tasks.workunit.client.0.smithi032.stderr: Exporting image: 100% complete...done. 2024-09-21T09:51:58.444 INFO:tasks.workunit.client.0.smithi032.stderr:++ stat /tmp/rbd_import_export_79662/sparse --format=%b 2024-09-21T09:51:58.445 INFO:tasks.workunit.client.0.smithi032.stderr:+ '[' 0 = 0 ']' 2024-09-21T09:51:58.445 INFO:tasks.workunit.client.0.smithi032.stderr:+ rbd rm sparse 2024-09-21T09:51:58.534 INFO:tasks.workunit.client.0.smithi032.stderr: Removing image: 100% complete...done. 2024-09-21T09:51:58.538 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm /tmp/rbd_import_export_79662/sparse /tmp/rbd_import_export_79662/sparse1 /tmp/rbd_import_export_79662/sparse2 /tmp/rbd_import_export_79662/sparse3 2024-09-21T09:51:58.540 INFO:tasks.workunit.client.0.smithi032.stderr:rm: cannot remove '/tmp/rbd_import_export_79662/sparse3': No such file or directory 2024-09-21T09:51:58.540 INFO:tasks.workunit.client.0.smithi032.stderr:+ true 2024-09-21T09:51:58.540 INFO:tasks.workunit.client.0.smithi032.stdout:OK 2024-09-21T09:51:58.541 INFO:tasks.workunit.client.0.smithi032.stderr:+ echo OK 2024-09-21T09:51:58.541 INFO:tasks.workunit.client.0.smithi032.stderr:+ rm -rf /tmp/rbd_import_export_79662 2024-09-21T09:51:58.542 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:51:58.542 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:51:58.615 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2024-09-21T09:51:58.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-21T09:51:58.912 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:58.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:51:58.941 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:51:58.941 DEBUG:teuthology.orchestra.run.smithi032:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:51:58.998 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:51:58.998 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T09:51:58.998 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2024-09-21T09:51:58.998 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:59.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:51:59 smithi045 ceph-mon[33087]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 398 op/s 2024-09-21T09:51:59.677 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:51:59 smithi032 ceph-mon[40061]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 398 op/s 2024-09-21T09:51:59.677 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:51:59 smithi032 ceph-mon[29191]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 398 op/s 2024-09-21T09:52:01.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:01 smithi045 ceph-mon[33087]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1018 KiB/s wr, 334 op/s 2024-09-21T09:52:01.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:01 smithi032 ceph-mon[29191]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1018 KiB/s wr, 334 op/s 2024-09-21T09:52:01.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:01 smithi032 ceph-mon[40061]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1018 KiB/s wr, 334 op/s 2024-09-21T09:52:03.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:02 smithi045 ceph-mon[33087]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:52:03.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:02 smithi032 ceph-mon[29191]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:52:03.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:02 smithi032 ceph-mon[40061]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:52:03.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:52:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:52:03] "GET /metrics HTTP/1.1" 200 33721 "" "Prometheus/2.43.0" 2024-09-21T09:52:04.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:03 smithi045 ceph-mon[33087]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1014 KiB/s rd, 1.4 MiB/s wr, 329 op/s 2024-09-21T09:52:04.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:03 smithi032 ceph-mon[29191]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1014 KiB/s rd, 1.4 MiB/s wr, 329 op/s 2024-09-21T09:52:04.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:03 smithi032 ceph-mon[40061]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1014 KiB/s rd, 1.4 MiB/s wr, 329 op/s 2024-09-21T09:52:05.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:05 smithi045 ceph-mon[33087]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 990 KiB/s rd, 1.4 MiB/s wr, 321 op/s 2024-09-21T09:52:05.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:05 smithi032 ceph-mon[40061]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 990 KiB/s rd, 1.4 MiB/s wr, 321 op/s 2024-09-21T09:52:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:05 smithi032 ceph-mon[29191]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 990 KiB/s rd, 1.4 MiB/s wr, 321 op/s 2024-09-21T09:52:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:07 smithi045 ceph-mon[33087]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2024-09-21T09:52:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:07.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:07 smithi032 ceph-mon[29191]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2024-09-21T09:52:07.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:07.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:07 smithi032 ceph-mon[40061]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2024-09-21T09:52:07.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:09 smithi032 ceph-mon[29191]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 410 KiB/s wr, 24 op/s 2024-09-21T09:52:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:09 smithi032 ceph-mon[40061]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 410 KiB/s wr, 24 op/s 2024-09-21T09:52:09.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:09 smithi045 ceph-mon[33087]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 410 KiB/s wr, 24 op/s 2024-09-21T09:52:11.425 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:11 smithi032 ceph-mon[29191]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 410 KiB/s wr, 24 op/s 2024-09-21T09:52:11.426 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:11 smithi032 ceph-mon[40061]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 410 KiB/s wr, 24 op/s 2024-09-21T09:52:11.453 INFO:teuthology.orchestra.run.smithi032.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:52:11.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:11 smithi045 ceph-mon[33087]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 410 KiB/s wr, 24 op/s 2024-09-21T09:52:12.095 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:52:12.306 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:52:12.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:12 smithi032 ceph-mon[40061]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:52:12.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:12 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:12.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:12 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:12.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:12 smithi032 ceph-mon[29191]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:52:12.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:12 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:12.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:12 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:12.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:12 smithi045 ceph-mon[33087]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:52:12.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:12 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:12.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:12 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:13.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:13 smithi032 ceph-mon[40061]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 366 KiB/s wr, 22 op/s 2024-09-21T09:52:13.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:13 smithi032 ceph-mon[29191]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 366 KiB/s wr, 22 op/s 2024-09-21T09:52:13.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:52:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:52:13] "GET /metrics HTTP/1.1" 200 33729 "" "Prometheus/2.43.0" 2024-09-21T09:52:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:13 smithi045 ceph-mon[33087]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 366 KiB/s wr, 22 op/s 2024-09-21T09:52:14.168 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:52:14.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:14 smithi032 ceph-mon[40061]: from='client.15495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:14.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:14 smithi032 ceph-mon[29191]: from='client.15495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (117s) 55s ago 2m 15.0M - 0.25.0 c8568f914cd2 75fcd3b3b5dd 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (2m) 89s ago 2m 82.5M - 9.4.7 954c08fa6188 419a19b40991 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (103s) 55s ago 102s 14.0M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (100s) 89s ago 100s 15.0M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,8765 running (6m) 89s ago 6m 437M - 18.2.4-871-g6a1bdf85 0adab276c97e ec216715bfa0 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:9283,8765 running (8m) 55s ago 8m 482M - 18.2.4-871-g6a1bdf85 0adab276c97e c1adf1e5a465 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (8m) 55s ago 8m 47.8M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 017caecda0d9 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (7m) 89s ago 7m 39.8M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:52:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (7m) 55s ago 7m 39.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 125675da4b84 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (2m) 55s ago 2m 14.2M - 1.5.0 0da6a335fe13 e4bf1511c584 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (2m) 89s ago 2m 7801k - 1.5.0 0da6a335fe13 49083320e62a 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (6m) 55s ago 6m 50.8M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (6m) 55s ago 6m 42.8M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (5m) 55s ago 5m 50.5M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (5m) 55s ago 5m 40.2M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (4m) 89s ago 4m 48.6M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (4m) 89s ago 4m 43.3M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (4m) 89s ago 4m 39.2M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (3m) 89s ago 3m 41.2M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:52:14.732 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (112s) 89s ago 3m 31.4M - 2.43.0 a07b618ecd1d b12ebc2bf34d 2024-09-21T09:52:14.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:14 smithi045 ceph-mon[33087]: from='client.15495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:15.100 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:52:15.100 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:52:15.100 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 15 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:52:15.101 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:52:15.416 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:52:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", 2024-09-21T09:52:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:52:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:52:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T09:52:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "", 2024-09-21T09:52:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 image", 2024-09-21T09:52:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:52:15.417 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:52:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: from='client.15507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:52:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2655422103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[29191]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: from='client.15507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/2655422103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:15 smithi032 ceph-mon[40061]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: from='client.15507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/2655422103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:15 smithi045 ceph-mon[33087]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:17.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:16 smithi045 ceph-mon[33087]: from='client.25412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:16 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:16 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:16 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:17.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:16 smithi032 ceph-mon[29191]: from='client.25412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:16 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:16 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:16 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:16 smithi032 ceph-mon[40061]: from='client.25412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:16 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:16 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:16 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:18.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:17 smithi045 ceph-mon[33087]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:52:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:17 smithi045 ceph-mon[33087]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:52:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:17 smithi045 ceph-mon[33087]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:52:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:17 smithi045 ceph-mon[33087]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi045 2024-09-21T09:52:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:17 smithi045 ceph-mon[33087]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[40061]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:52:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[40061]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:52:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[40061]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:52:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[40061]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi045 2024-09-21T09:52:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[40061]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[29191]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:52:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[29191]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:52:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[29191]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:52:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[29191]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi045 2024-09-21T09:52:18.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:17 smithi032 ceph-mon[29191]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:19.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:19 smithi045 ceph-mon[33087]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:19.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:19 smithi032 ceph-mon[29191]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:19.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:19 smithi032 ceph-mon[40061]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:21.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:21 smithi045 ceph-mon[33087]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:21.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:21 smithi032 ceph-mon[40061]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:21 smithi032 ceph-mon[29191]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:22.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:22 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:22.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:22 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:22.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:22 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:23.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:23 smithi045 ceph-mon[33087]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:23.626 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:23 smithi032 ceph-mon[29191]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:23.626 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:23 smithi032 ceph-mon[40061]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:23.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:52:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:52:23] "GET /metrics HTTP/1.1" 200 33728 "" "Prometheus/2.43.0" 2024-09-21T09:52:25.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:25 smithi045 ceph-mon[33087]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:25.555 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:25 smithi032 ceph-mon[29191]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:25.555 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:25 smithi032 ceph-mon[40061]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:27.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:27 smithi045 ceph-mon[33087]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:27 smithi032 ceph-mon[29191]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:27.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:27 smithi032 ceph-mon[40061]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:29.509 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:29 smithi032 ceph-mon[40061]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:29.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:29 smithi032 ceph-mon[29191]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:29.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:29 smithi045 ceph-mon[33087]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:31.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:31 smithi045 ceph-mon[33087]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:31.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:31 smithi032 ceph-mon[40061]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:31.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:31 smithi032 ceph-mon[29191]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:33.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:33 smithi045 ceph-mon[33087]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:33.625 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:33 smithi032 ceph-mon[29191]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:33.625 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:33 smithi032 ceph-mon[40061]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:33.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:52:33 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:52:33] "GET /metrics HTTP/1.1" 200 33728 "" "Prometheus/2.43.0" 2024-09-21T09:52:35.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:35 smithi045 ceph-mon[33087]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:35.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:35 smithi032 ceph-mon[40061]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:35.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:35 smithi032 ceph-mon[29191]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:37 smithi045 ceph-mon[33087]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:37 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:37.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:37 smithi032 ceph-mon[29191]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:37.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:37 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:37.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:37 smithi032 ceph-mon[40061]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:37.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:37 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:39.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:39 smithi045 ceph-mon[33087]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:39.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:39 smithi032 ceph-mon[29191]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:39.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:39 smithi032 ceph-mon[40061]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:41.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:41 smithi045 ceph-mon[33087]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:41.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:41 smithi032 ceph-mon[40061]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:41 smithi032 ceph-mon[29191]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:43.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:43 smithi045 ceph-mon[33087]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:43.625 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:43 smithi032 ceph-mon[29191]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:43.625 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:43 smithi032 ceph-mon[40061]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:43.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:52:43 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:52:43] "GET /metrics HTTP/1.1" 200 33730 "" "Prometheus/2.43.0" 2024-09-21T09:52:45.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:45 smithi045 ceph-mon[33087]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:45 smithi032 ceph-mon[29191]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:45 smithi032 ceph-mon[40061]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:45.729 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:52:46.282 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:52:46.890 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (2m) 87s ago 3m 15.0M - 0.25.0 c8568f914cd2 75fcd3b3b5dd 2024-09-21T09:52:46.890 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (3m) 31s ago 3m 90.8M - 9.4.7 954c08fa6188 419a19b40991 2024-09-21T09:52:46.891 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (2m) 87s ago 2m 14.0M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:52:46.891 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (2m) 31s ago 2m 15.4M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:52:46.891 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,8765 running (7m) 31s ago 7m 438M - 18.2.4-871-g6a1bdf85 0adab276c97e ec216715bfa0 2024-09-21T09:52:46.891 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:9283,8765 running (9m) 87s ago 9m 482M - 18.2.4-871-g6a1bdf85 0adab276c97e c1adf1e5a465 2024-09-21T09:52:46.891 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (9m) 87s ago 9m 47.8M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 017caecda0d9 2024-09-21T09:52:46.891 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (7m) 31s ago 7m 42.1M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:52:46.891 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (7m) 87s ago 7m 39.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 125675da4b84 2024-09-21T09:52:46.891 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (2m) 87s ago 2m 14.2M - 1.5.0 0da6a335fe13 e4bf1511c584 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (2m) 31s ago 2m 15.0M - 1.5.0 0da6a335fe13 49083320e62a 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (7m) 87s ago 6m 50.8M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (6m) 87s ago 6m 42.8M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (6m) 87s ago 6m 50.5M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (5m) 87s ago 5m 40.2M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (5m) 31s ago 5m 51.9M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (5m) 31s ago 5m 46.5M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (4m) 31s ago 4m 45.4M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (4m) 31s ago 4m 47.6M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:52:46.892 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (2m) 31s ago 3m 44.4M - 2.43.0 a07b618ecd1d b12ebc2bf34d 2024-09-21T09:52:46.893 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:52:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:52:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:52:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:52:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:52:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 15 2024-09-21T09:52:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:52:46.895 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:52:46.908 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:52:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:52:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:52:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:52:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T09:52:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "0/2 daemons upgraded", 2024-09-21T09:52:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading mgr daemons", 2024-09-21T09:52:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:52:46.909 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:52:46.909 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[40061]: from='client.25418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:46.909 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[40061]: Upgrade: Updating mgr.x 2024-09-21T09:52:46.909 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:46.909 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:52:46.909 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:52:46.910 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:46.910 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[40061]: Deploying daemon mgr.x on smithi045 2024-09-21T09:52:46.910 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[40061]: from='client.25424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:46.910 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/2884348259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:46.910 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[29191]: from='client.25418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:46.910 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[29191]: Upgrade: Updating mgr.x 2024-09-21T09:52:46.910 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:46.910 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:52:46.910 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:52:46.911 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:46.911 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[29191]: Deploying daemon mgr.x on smithi045 2024-09-21T09:52:46.911 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[29191]: from='client.25424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:46.911 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:46 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/2884348259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:46 smithi045 ceph-mon[33087]: from='client.25418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:46 smithi045 ceph-mon[33087]: Upgrade: Updating mgr.x 2024-09-21T09:52:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:46 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:46 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:52:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:46 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:52:47.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:46 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:47.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:46 smithi045 ceph-mon[33087]: Deploying daemon mgr.x on smithi045 2024-09-21T09:52:47.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:46 smithi045 ceph-mon[33087]: from='client.25424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:47.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:46 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/2884348259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:48.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:47 smithi045 systemd[1]: Stopping Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:52:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:47 smithi045 ceph-mon[33087]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:47 smithi045 ceph-mon[33087]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:47 smithi045 ceph-mon[33087]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:48.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:47 smithi032 ceph-mon[29191]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:47 smithi032 ceph-mon[29191]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:47 smithi032 ceph-mon[29191]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:47 smithi032 ceph-mon[40061]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:47 smithi032 ceph-mon[40061]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:47 smithi032 ceph-mon[40061]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:48.403 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:48 smithi045 podman[65828]: 2024-09-21 09:52:48.081564975 +0000 UTC m=+0.156947815 container died ec216715bfa0c7a82d47671773ff553ba994973f81efd86470d575247c108ea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:52:48.784 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:48 smithi045 podman[65828]: 2024-09-21 09:52:48.584961835 +0000 UTC m=+0.660344705 container cleanup ec216715bfa0c7a82d47671773ff553ba994973f81efd86470d575247c108ea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T09:52:48.785 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:48 smithi045 bash[65828]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:52:49.242 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:48 smithi045 podman[65841]: 2024-09-21 09:52:48.794670355 +0000 UTC m=+0.716737973 container remove ec216715bfa0c7a82d47671773ff553ba994973f81efd86470d575247c108ea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=reef-6a1bdf8, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T09:52:49.242 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:48 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:52:49.242 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:52:49.242 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 systemd[1]: Stopped Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:52:49.242 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x.service: Consumed 11.320s CPU time. 2024-09-21T09:52:49.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:49 smithi045 ceph-mon[33087]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:49.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 systemd[1]: Starting Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:52:49.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 podman[65971]: 2024-09-21 09:52:49.367800975 +0000 UTC m=+0.016772254 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:52:49.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 podman[65971]: 2024-09-21 09:52:49.488288895 +0000 UTC m=+0.137260206 container create 5cc590ae518b644bf411dd4810551e71e5776efd6c1e385a0347b9c0a1efb39a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-21T09:52:49.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:49 smithi032 ceph-mon[29191]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:49.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:49 smithi032 ceph-mon[40061]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:50.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 podman[65971]: 2024-09-21 09:52:49.682385062 +0000 UTC m=+0.331356341 container init 5cc590ae518b644bf411dd4810551e71e5776efd6c1e385a0347b9c0a1efb39a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-62da54b) 2024-09-21T09:52:50.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 podman[65971]: 2024-09-21 09:52:49.68546882 +0000 UTC m=+0.334440098 container start 5cc590ae518b644bf411dd4810551e71e5776efd6c1e385a0347b9c0a1efb39a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T09:52:50.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 ceph-mgr[65990]: -- 172.21.15.45:0/123898376 <== mon.1 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b8c91870e0 con 0x55b8c9165000 2024-09-21T09:52:50.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 bash[65971]: 5cc590ae518b644bf411dd4810551e71e5776efd6c1e385a0347b9c0a1efb39a 2024-09-21T09:52:50.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:49 smithi045 systemd[1]: Started Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:52:51.142 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:50 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:51.142 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:50 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:51.142 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:50 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:50 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:50 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:50 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:50 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:50 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:50 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:52.105 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:51 smithi032 ceph-mon[40061]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:52.106 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:51 smithi032 ceph-mon[29191]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:52.176 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:51 smithi045 ceph-mon[33087]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:52 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:52 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:52 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:53.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:52 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:53.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:52 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:53.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:52 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:53.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:52 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:53.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:52 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:52 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:52:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:53 smithi032 ceph-mon[29191]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:53 smithi032 ceph-mon[40061]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:53.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:52:53 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:52:53] "GET /metrics HTTP/1.1" 200 33732 "" "Prometheus/2.43.0" 2024-09-21T09:52:54.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:53 smithi045 ceph-mon[33087]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-mon[33087]: Standby manager daemon x restarted 2024-09-21T09:52:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-mon[33087]: Standby manager daemon x started 2024-09-21T09:52:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:52:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:52:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:52:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:52:55.286 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:52:55] ENGINE Bus STARTING 2024-09-21T09:52:55.286 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: CherryPy Checker: 2024-09-21T09:52:55.286 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: The Application mounted at '' has an empty config. 2024-09-21T09:52:55.286 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:52:55.286 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:52:55] ENGINE Serving on http://:::9283 2024-09-21T09:52:55.286 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:52:55 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:52:55] ENGINE Bus STARTED 2024-09-21T09:52:55.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[29191]: Standby manager daemon x restarted 2024-09-21T09:52:55.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[29191]: Standby manager daemon x started 2024-09-21T09:52:55.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:52:55.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:52:55.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:52:55.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:52:55.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[40061]: Standby manager daemon x restarted 2024-09-21T09:52:55.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[40061]: Standby manager daemon x started 2024-09-21T09:52:55.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:52:55.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:52:55.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:52:55.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:55 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:52:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:56 smithi045 ceph-mon[33087]: mgrmap e21: y(active, since 3m), standbys: x 2024-09-21T09:52:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:56 smithi045 ceph-mon[33087]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:56.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:56 smithi032 ceph-mon[40061]: mgrmap e21: y(active, since 3m), standbys: x 2024-09-21T09:52:56.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:56 smithi032 ceph-mon[40061]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:56 smithi032 ceph-mon[29191]: mgrmap e21: y(active, since 3m), standbys: x 2024-09-21T09:52:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:56 smithi032 ceph-mon[29191]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:57.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:57 smithi045 ceph-mon[33087]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:57.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:57 smithi032 ceph-mon[40061]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:57.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:57 smithi032 ceph-mon[29191]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:59.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:52:59 smithi032 ceph-mon[29191]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:59.509 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:52:59 smithi032 ceph-mon[40061]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:52:59.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:52:59 smithi045 ceph-mon[33087]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:01.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:01 smithi045 ceph-mon[33087]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:01.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:01 smithi032 ceph-mon[40061]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:01.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:01 smithi032 ceph-mon[29191]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:03.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:03 smithi045 ceph-mon[33087]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:03.625 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:03 smithi032 ceph-mon[40061]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:03.626 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:03 smithi032 ceph-mon[29191]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:03.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:53:03] "GET /metrics HTTP/1.1" 200 33732 "" "Prometheus/2.43.0" 2024-09-21T09:53:06.176 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:05 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.177 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:05 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.177 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:05 smithi045 ceph-mon[33087]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:06.177 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:05 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.177 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:05 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[29191]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[40061]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:05 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:07.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:07 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:07 smithi045 ceph-mon[33087]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:07.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:07 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:07.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:07 smithi032 ceph-mon[40061]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:07.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:07 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:07.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:07 smithi032 ceph-mon[29191]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:09.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:08 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:53:08] ENGINE Bus STOPPING 2024-09-21T09:53:09.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:08 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ignoring --setuser ceph since I am not root 2024-09-21T09:53:09.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:08 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ignoring --setgroup ceph since I am not root 2024-09-21T09:53:09.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:09.023+0000 7f9f3edb2040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:53:09.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:09.082+0000 7f9f3edb2040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:53:09.534 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:53:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:53:09.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:53:09] ENGINE Bus STOPPED 2024-09-21T09:53:09.605 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:09.239+0000 7f9f3edb2040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:53:09.606 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:09.307+0000 7f9f3edb2040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:53:09.606 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:09.458+0000 7f9f3edb2040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:53:09.931 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:09.604+0000 7f9f3edb2040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:53:09.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:09.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:09.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:09.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:09.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:09.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:09.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:53:09.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: Activating manager daemon x 2024-09-21T09:53:09.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:53:09.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:53:09.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: mgrmap e22: x(active, starting, since 0.0076472s) 2024-09-21T09:53:09.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:53:09.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:53:09.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:53:09.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:53:09.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:53:09.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:53:09.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:53:09.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:53:09.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:53:09.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:53:09.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:53:09.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:53:09.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:53:09.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:53:09.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:53:09.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:53:09.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:53:09.936 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[29191]: Manager daemon x is now available 2024-09-21T09:53:09.936 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:09.936 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:09.936 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:09.936 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:09.936 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:09.936 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:09.937 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:53:09.937 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: Activating manager daemon x 2024-09-21T09:53:09.937 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:53:09.937 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:53:09.937 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: mgrmap e22: x(active, starting, since 0.0076472s) 2024-09-21T09:53:09.937 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:53:09.937 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:53:09.938 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:53:09.938 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:53:09.938 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:53:09.938 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:53:09.938 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:53:09.938 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:53:09.938 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:53:09.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:53:09.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:53:09.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:53:09.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:53:09.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:53:09.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:53:09.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:53:09.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:53:09.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-mon[40061]: Manager daemon x is now available 2024-09-21T09:53:09.999 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:53:09] ENGINE Bus STARTING 2024-09-21T09:53:09.999 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.997+0000 7fdca9e12640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: Activating manager daemon x 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.24335 172.21.15.32:0/765233215' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: mgrmap e22: x(active, starting, since 0.0076472s) 2024-09-21T09:53:10.000 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:53:10.001 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:53:10.002 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:53:10.002 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-mon[33087]: Manager daemon x is now available 2024-09-21T09:53:10.187 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:09.929+0000 7f9f3edb2040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:53:10.188 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:09.995+0000 7f9f3edb2040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:53:10.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.997+0000 7fdca9e12640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.997+0000 7fdca9e12640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.997+0000 7fdca9e12640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.997+0000 7fdca9e12640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.998+0000 7fdcace18640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.998+0000 7fdcace18640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.998+0000 7fdcace18640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.998+0000 7fdcace18640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:09 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:09.998+0000 7fdcace18640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:10.286 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:53:10] ENGINE Serving on http://:::9283 2024-09-21T09:53:10.286 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:53:10] ENGINE Bus STARTED 2024-09-21T09:53:10.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:10.186+0000 7f9f3edb2040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:53:11.173 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:11 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:11.103+0000 7f9f3edb2040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: mgrmap e23: x(active, since 1.00852s) 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: Checking for cert/key for grafana.a 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: Migrating grafana.a cert to cert store 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: Migrating grafana.a key to cert store 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:11.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: Deploying cephadm binary to smithi045 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[29191]: Deploying cephadm binary to smithi032 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: mgrmap e23: x(active, since 1.00852s) 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: Checking for cert/key for grafana.a 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: Migrating grafana.a cert to cert store 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: Migrating grafana.a key to cert store 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:11.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:11.177 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:11.177 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:11.177 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: Deploying cephadm binary to smithi045 2024-09-21T09:53:11.177 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:10 smithi032 ceph-mon[40061]: Deploying cephadm binary to smithi032 2024-09-21T09:53:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: mgrmap e23: x(active, since 1.00852s) 2024-09-21T09:53:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: Checking for cert/key for grafana.a 2024-09-21T09:53:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: Migrating grafana.a cert to cert store 2024-09-21T09:53:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: Migrating grafana.a key to cert store 2024-09-21T09:53:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: Deploying cephadm binary to smithi045 2024-09-21T09:53:11.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:10 smithi045 ceph-mon[33087]: Deploying cephadm binary to smithi032 2024-09-21T09:53:11.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:11 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:11.171+0000 7f9f3edb2040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:53:11.885 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:11.799+0000 7fdc91b67640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:53:11] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:11.885 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: Traceback (most recent call last): 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: self._connections.run(self.expiration_interval) 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: self._run(expiration_interval) 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: s = self.context.wrap_socket( 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: return self.sslsocket_class._create( 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:11.886 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: self.do_handshake() 2024-09-21T09:53:11.887 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:11.887 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: self._sslobj.do_handshake() 2024-09-21T09:53:11.887 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:11.887 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:12.196 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:11 smithi032 ceph-mon[40061]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:12.196 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:11 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:11] ENGINE Bus STARTING 2024-09-21T09:53:12.196 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:11 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:11] ENGINE Serving on http://172.21.15.45:8765 2024-09-21T09:53:12.197 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:11 smithi032 ceph-mon[29191]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:12.197 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:11 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:11] ENGINE Bus STARTING 2024-09-21T09:53:12.197 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:11 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:11] ENGINE Serving on http://172.21.15.45:8765 2024-09-21T09:53:12.197 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:11 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:11.922+0000 7f9f3edb2040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:53:12.197 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.044+0000 7f9f3edb2040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:53:12.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-mon[33087]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:12.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:11] ENGINE Bus STARTING 2024-09-21T09:53:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:11 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:11] ENGINE Serving on http://172.21.15.45:8765 2024-09-21T09:53:12.456 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.194+0000 7f9f3edb2040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:53:12.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.261+0000 7f9f3edb2040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:53:12.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.426+0000 7f9f3edb2040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:53:12.775 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.497+0000 7f9f3edb2040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:53:12.775 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.559+0000 7f9f3edb2040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:53:12.775 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.724+0000 7f9f3edb2040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:53:13.076 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:11] ENGINE Serving on https://172.21.15.45:7150 2024-09-21T09:53:13.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:11] ENGINE Bus STARTED 2024-09-21T09:53:13.345 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:11] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:13.346 INFO:journalctl@ceph.mon.c.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:53:13.348 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:13.348 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:53:13.348 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:13.348 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:53:13.348 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:13.348 INFO:journalctl@ceph.mon.c.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:13.348 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:13.349 INFO:journalctl@ceph.mon.c.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:13.349 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:13.349 INFO:journalctl@ceph.mon.c.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:53:13.349 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:13.349 INFO:journalctl@ceph.mon.c.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:53:13.349 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:13.349 INFO:journalctl@ceph.mon.c.smithi032.stdout: self.do_handshake() 2024-09-21T09:53:13.350 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:13.350 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:53:13.350 INFO:journalctl@ceph.mon.c.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:13.350 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-mon[40061]: mgrmap e24: x(active, since 3s) 2024-09-21T09:53:13.354 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:11] ENGINE Serving on https://172.21.15.45:7150 2024-09-21T09:53:13.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:11] ENGINE Bus STARTED 2024-09-21T09:53:13.355 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:11] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:13.355 INFO:journalctl@ceph.mon.a.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:53:13.355 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:13.355 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:53:13.355 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:13.356 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:53:13.356 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:13.356 INFO:journalctl@ceph.mon.a.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:13.356 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:13.356 INFO:journalctl@ceph.mon.a.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:13.356 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:13.356 INFO:journalctl@ceph.mon.a.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:53:13.356 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:13.357 INFO:journalctl@ceph.mon.a.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:53:13.357 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:13.357 INFO:journalctl@ceph.mon.a.smithi032.stdout: self.do_handshake() 2024-09-21T09:53:13.357 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:13.357 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:53:13.357 INFO:journalctl@ceph.mon.a.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:13.357 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-mon[29191]: mgrmap e24: x(active, since 3s) 2024-09-21T09:53:13.358 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.794+0000 7f9f3edb2040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:53:13.358 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.863+0000 7f9f3edb2040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:53:13.358 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:12.950+0000 7f9f3edb2040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:53:13.358 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:13.012+0000 7f9f3edb2040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:53:13.358 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:13.073+0000 7f9f3edb2040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:53:13.359 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:12 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:11] ENGINE Serving on https://172.21.15.45:7150 2024-09-21T09:53:13.359 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:12 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:11] ENGINE Bus STARTED 2024-09-21T09:53:13.359 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:12 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:11] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:13.359 INFO:journalctl@ceph.mon.b.smithi045.stdout: Traceback (most recent call last): 2024-09-21T09:53:13.359 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:13.359 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:53:13.359 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:13.359 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._run(expiration_interval) 2024-09-21T09:53:13.359 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: s = self.context.wrap_socket( 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: return self.sslsocket_class._create( 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: self.do_handshake() 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._sslobj.do_handshake() 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:13.360 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:12 smithi045 ceph-mon[33087]: mgrmap e24: x(active, since 3s) 2024-09-21T09:53:14.033 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:13 smithi032 ceph-mon[29191]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:14.033 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:13 smithi032 ceph-mon[40061]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:14.033 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:13.923+0000 7f9f3edb2040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:53:14.034 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:53:13] ENGINE Bus STARTING 2024-09-21T09:53:14.034 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: CherryPy Checker: 2024-09-21T09:53:14.034 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: The Application mounted at '' has an empty config. 2024-09-21T09:53:14.034 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: 2024-09-21T09:53:14.282 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:13 smithi045 ceph-mon[33087]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:14.409 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:53:14] ENGINE Serving on http://:::9283 2024-09-21T09:53:14.409 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: [21/Sep/2024:09:53:14] ENGINE Bus STARTED 2024-09-21T09:53:14.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[29191]: mgrmap e25: x(active, since 5s) 2024-09-21T09:53:14.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[29191]: Standby manager daemon y started 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[40061]: mgrmap e25: x(active, since 5s) 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[40061]: Standby manager daemon y started 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:53:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:14 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:14 smithi045 ceph-mon[33087]: mgrmap e25: x(active, since 5s) 2024-09-21T09:53:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:14 smithi045 ceph-mon[33087]: Standby manager daemon y started 2024-09-21T09:53:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:14 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:53:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:14 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:14 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:53:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:14 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.32:0/526478370' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 614 B/s wr, 14 op/s 2024-09-21T09:53:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: mgrmap e26: x(active, since 6s), standbys: y 2024-09-21T09:53:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:53:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi032 to 2481M 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 614 B/s wr, 14 op/s 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: mgrmap e26: x(active, since 6s), standbys: y 2024-09-21T09:53:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi032 to 2481M 2024-09-21T09:53:16.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:15 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 614 B/s wr, 14 op/s 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: mgrmap e26: x(active, since 6s), standbys: y 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:16.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi032 to 2481M 2024-09-21T09:53:16.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:15 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.235 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:53:17.334 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:17 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.334 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:17 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.334 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:17 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.334 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:17 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:17 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:17 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:17 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:17 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:17 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:17 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:17 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:17 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:17.822 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:53:17.822 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (3m) 2s ago 3m 28.4M - 0.25.0 c8568f914cd2 75fcd3b3b5dd 2024-09-21T09:53:17.822 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (3m) 1s ago 3m 88.5M - 9.4.7 954c08fa6188 419a19b40991 2024-09-21T09:53:17.822 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (2m) 2s ago 2m 15.6M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (2m) 1s ago 2m 16.1M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (28s) 1s ago 7m 497M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5cc590ae518b 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:9283,8765 running (9m) 2s ago 9m 311M - 18.2.4-871-g6a1bdf85 0adab276c97e c1adf1e5a465 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (9m) 2s ago 9m 52.1M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 017caecda0d9 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (8m) 1s ago 8m 44.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (8m) 2s ago 8m 43.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 125675da4b84 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (3m) 2s ago 3m 16.0M - 1.5.0 0da6a335fe13 e4bf1511c584 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (3m) 1s ago 3m 15.1M - 1.5.0 0da6a335fe13 49083320e62a 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (7m) 2s ago 7m 57.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (7m) 2s ago 7m 51.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (6m) 2s ago 6m 56.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (6m) 2s ago 6m 44.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (6m) 1s ago 6m 52.2M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:53:17.823 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (5m) 1s ago 5m 47.1M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:53:17.824 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (5m) 1s ago 5m 46.0M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:53:17.824 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (4m) 1s ago 4m 48.4M 4017M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:53:17.824 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (2m) 1s ago 4m 51.6M - 2.43.0 a07b618ecd1d b12ebc2bf34d 2024-09-21T09:53:18.180 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:53:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:53:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:53:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:53:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:53:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 1, 2024-09-21T09:53:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:53:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:53:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:53:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:53:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:53:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:53:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:53:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:53:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:53:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 14, 2024-09-21T09:53:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:53:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:53:18.182 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:53:18.482 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:53:18.482 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:53:18.482 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:53:18.482 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:53:18.482 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T09:53:18.482 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "1/2 daemons upgraded", 2024-09-21T09:53:18.482 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "", 2024-09-21T09:53:18.482 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:53:18.482 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:53:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:18 smithi032 ceph-mon[29191]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 438 B/s wr, 10 op/s 2024-09-21T09:53:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:18 smithi032 ceph-mon[29191]: from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:18 smithi032 ceph-mon[29191]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:18 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/596327549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:18 smithi032 ceph-mon[40061]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 438 B/s wr, 10 op/s 2024-09-21T09:53:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:18 smithi032 ceph-mon[40061]: from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:18 smithi032 ceph-mon[40061]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:18 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/596327549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:18 smithi045 ceph-mon[33087]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 438 B/s wr, 10 op/s 2024-09-21T09:53:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:18 smithi045 ceph-mon[33087]: from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:18 smithi045 ceph-mon[33087]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:18 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/596327549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:19.851 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:19 smithi045 ceph-mon[33087]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:19.851 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:19 smithi045 ceph-mon[33087]: from='client.25514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:19.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:19 smithi032 ceph-mon[40061]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:19 smithi032 ceph-mon[40061]: from='client.25514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:19 smithi032 ceph-mon[29191]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:19 smithi032 ceph-mon[29191]: from='client.25514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:20.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:20 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:53:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:20 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Adjusting osd_memory_target on smithi045 to 2481M 2024-09-21T09:53:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:53:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:53:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:22.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:21 smithi045 ceph-mon[33087]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:53:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Adjusting osd_memory_target on smithi045 to 2481M 2024-09-21T09:53:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:53:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:53:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:22.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:22.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:22.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:22.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[29191]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:53:22.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Adjusting osd_memory_target on smithi045 to 2481M 2024-09-21T09:53:22.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:53:22.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:53:22.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:22.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:22.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:22.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:22.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:22.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:22.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:21 smithi032 ceph-mon[40061]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:53:22.960 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:22 smithi045 ceph-mon[33087]: Reconfiguring daemon prometheus.a on smithi045 2024-09-21T09:53:22.960 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:22 smithi045 ceph-mon[33087]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-09-21T09:53:22.961 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:22.665Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T09:53:22.961 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:22.665Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T09:53:22.961 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:22.668Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T09:53:22.961 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:22.669Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T09:53:22.961 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:22.669Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T09:53:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:22 smithi032 ceph-mon[40061]: Reconfiguring daemon prometheus.a on smithi045 2024-09-21T09:53:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:22 smithi032 ceph-mon[40061]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-09-21T09:53:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:22 smithi032 ceph-mon[29191]: Reconfiguring daemon prometheus.a on smithi045 2024-09-21T09:53:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:22 smithi032 ceph-mon[29191]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-09-21T09:53:23.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[29418]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:53:24.034 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:23 smithi045 systemd[1]: Stopping Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:24.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.110Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:53:24.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.110Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:53:24.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:53:24.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:53:24.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:53:24.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:53:24.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:53:24.388 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:53:24.388 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:53:24.388 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:53:24.388 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:53:24.388 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[62725]: ts=2024-09-21T09:53:24.111Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:53:24.388 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 podman[73625]: 2024-09-21 09:53:24.121122624 +0000 UTC m=+0.129586679 container died b12ebc2bf34d975d6dec270fc7c334bc4beb06e902398f9d6e038aaf88f06ee3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:24.716 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 podman[73625]: 2024-09-21 09:53:24.559568191 +0000 UTC m=+0.568032129 container cleanup b12ebc2bf34d975d6dec270fc7c334bc4beb06e902398f9d6e038aaf88f06ee3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:24.716 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 bash[73625]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:53:24.973 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 podman[73637]: 2024-09-21 09:53:24.770436904 +0000 UTC m=+0.651442447 container remove b12ebc2bf34d975d6dec270fc7c334bc4beb06e902398f9d6e038aaf88f06ee3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:24.973 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:24 smithi045 ceph-mon[33087]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-21T09:53:25.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:24 smithi032 ceph-mon[29191]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-21T09:53:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:24 smithi032 ceph-mon[40061]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-21T09:53:25.248 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:53:25.248 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 systemd[1]: Stopped Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:25.248 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.661s CPU time. 2024-09-21T09:53:25.248 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:24 smithi045 systemd[1]: Starting Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:25.248 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 podman[73730]: 2024-09-21 09:53:25.096960004 +0000 UTC m=+0.017392581 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:53:25.248 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 podman[73730]: 2024-09-21 09:53:25.246734106 +0000 UTC m=+0.167166673 container create 6a0f34cf349510247fc9e958083fde8230cc8a45e53434a3ea0ee0bf79ba9da3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:25.511 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 podman[73730]: 2024-09-21 09:53:25.482596094 +0000 UTC m=+0.403028669 container init 6a0f34cf349510247fc9e958083fde8230cc8a45e53434a3ea0ee0bf79ba9da3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:25.511 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 podman[73730]: 2024-09-21 09:53:25.485519188 +0000 UTC m=+0.405951756 container start 6a0f34cf349510247fc9e958083fde8230cc8a45e53434a3ea0ee0bf79ba9da3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.509Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.509Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.509Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi045 (none))" 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.509Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.509Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.511Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.511Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.513Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.513Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.515Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:53:25.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.515Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.058µs 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.515Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.516Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.523Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.525Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.525Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.103µs wal_replay_duration=9.608374ms wbl_replay_duration=120ns total_replay_duration=9.650747ms 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.526Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.526Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.526Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.548Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.890013ms db_storage=747ns remote_storage=1.109µs web_handler=351ns query_engine=536ns scrape=357.194µs scrape_sd=121.615µs notify=15.492µs notify_sd=61.652µs rules=20.881833ms tracing=9.129µs 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.548Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:25.548Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 bash[73730]: 6a0f34cf349510247fc9e958083fde8230cc8a45e53434a3ea0ee0bf79ba9da3 2024-09-21T09:53:25.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:25 smithi045 systemd[1]: Started Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:26.007 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:26.007 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:26.008 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:26.008 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:26.008 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:26.008 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:26.008 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:26.008 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:26.008 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:26.008 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:25 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:26.868 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:26 smithi045 ceph-mon[33087]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-21T09:53:26.868 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:26 smithi045 ceph-mon[33087]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:26.869 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:26 smithi045 ceph-mon[33087]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:26.869 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:26 smithi045 ceph-mon[33087]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:27.018 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:26 smithi032 ceph-mon[29191]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-21T09:53:27.019 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:26 smithi032 ceph-mon[29191]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:27.019 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:26 smithi032 ceph-mon[29191]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:27.019 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:26 smithi032 ceph-mon[29191]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:27.019 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:26 smithi032 ceph-mon[40061]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-21T09:53:27.019 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:26 smithi032 ceph-mon[40061]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:27.019 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:26 smithi032 ceph-mon[40061]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:27.019 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:26 smithi032 ceph-mon[40061]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:28.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:28 smithi045 ceph-mon[33087]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:28 smithi045 ceph-mon[33087]: Upgrade: Updating mgr.y 2024-09-21T09:53:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:28 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:28 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:28 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:28 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:53:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:28 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:28 smithi045 ceph-mon[33087]: Deploying daemon mgr.y on smithi032 2024-09-21T09:53:28.443 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[40061]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:28.443 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[40061]: Upgrade: Updating mgr.y 2024-09-21T09:53:28.443 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:28.443 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:28.443 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:28.444 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:53:28.444 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:28.444 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[40061]: Deploying daemon mgr.y on smithi032 2024-09-21T09:53:28.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[29191]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:28.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[29191]: Upgrade: Updating mgr.y 2024-09-21T09:53:28.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:28.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:28.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:28.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:53:28.445 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:28.445 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:28 smithi032 ceph-mon[29191]: Deploying daemon mgr.y on smithi032 2024-09-21T09:53:29.028 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:28 smithi032 systemd[1]: Stopping Ceph mgr.y for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:29.332 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:29 smithi032 podman[85070]: 2024-09-21 09:53:29.026579973 +0000 UTC m=+0.147183092 container died c1adf1e5a4653876430bf5157f305dd3547a4aeb9fc84fa5cafe9da754d50933 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, RELEASE=reef-6a1bdf8, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T09:53:29.682 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:29 smithi032 podman[85070]: 2024-09-21 09:53:29.437976769 +0000 UTC m=+0.558579886 container cleanup c1adf1e5a4653876430bf5157f305dd3547a4aeb9fc84fa5cafe9da754d50933 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, org.label-schema.license=GPLv2, RELEASE=reef-6a1bdf8, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T09:53:29.682 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:29 smithi032 bash[85070]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y 2024-09-21T09:53:29.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:29 smithi032 podman[85081]: 2024-09-21 09:53:29.681012288 +0000 UTC m=+0.650384967 container remove c1adf1e5a4653876430bf5157f305dd3547a4aeb9fc84fa5cafe9da754d50933 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:53:29.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:29 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:53:29.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:29 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y.service: Failed with result 'exit-code'. 2024-09-21T09:53:29.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:29 smithi032 systemd[1]: Stopped Ceph mgr.y for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:29.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:29 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y.service: Consumed 47.180s CPU time. 2024-09-21T09:53:30.304 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:30 smithi032 ceph-mon[29191]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:30.305 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:30 smithi032 systemd[1]: Starting Ceph mgr.y for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:30.305 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:30 smithi032 podman[85214]: 2024-09-21 09:53:30.160936977 +0000 UTC m=+0.020823462 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:53:30.305 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:30 smithi032 podman[85214]: 2024-09-21 09:53:30.265228553 +0000 UTC m=+0.125115023 container create 581ec7ea9f79414c143dd4fa322421d2e6fc5d11699b272c86eff89c842de324 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:53:30.305 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:30 smithi032 ceph-mon[40061]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:30.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:30 smithi045 ceph-mon[33087]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:30.577 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:30 smithi032 podman[85214]: 2024-09-21 09:53:30.509620175 +0000 UTC m=+0.369506656 container init 581ec7ea9f79414c143dd4fa322421d2e6fc5d11699b272c86eff89c842de324 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T09:53:30.577 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:30 smithi032 podman[85214]: 2024-09-21 09:53:30.512699852 +0000 UTC m=+0.372586330 container start 581ec7ea9f79414c143dd4fa322421d2e6fc5d11699b272c86eff89c842de324 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T09:53:30.578 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:30 smithi032 ceph-mgr[85239]: -- 172.21.15.32:0/1252321142 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555820f830e0 con 0x555820f60800 2024-09-21T09:53:30.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:30 smithi032 bash[85214]: 581ec7ea9f79414c143dd4fa322421d2e6fc5d11699b272c86eff89c842de324 2024-09-21T09:53:30.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:30 smithi032 systemd[1]: Started Ceph mgr.y for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:31.239 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:53:30] "GET /metrics HTTP/1.1" 200 33802 "" "Prometheus/2.43.0" 2024-09-21T09:53:31.923 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:31 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:31.923 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:31 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:31.923 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:31 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:31.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:31 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:31.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:31 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:31.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:31 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:31 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:31 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:31 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:32.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:32 smithi032 ceph-mon[29191]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:32 smithi032 ceph-mon[40061]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:32.993 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:32 smithi045 ceph-mon[33087]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:34.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:34 smithi032 ceph-mon[40061]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:34 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:34 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:34 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:34 smithi032 ceph-mon[29191]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:34 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:34 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:34 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:34 smithi045 ceph-mon[33087]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:34 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:34 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:34 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:35.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:36.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:35 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:35 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:35 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:36.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:53:35] ENGINE Bus STARTING 2024-09-21T09:53:36.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: CherryPy Checker: 2024-09-21T09:53:36.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: The Application mounted at '' has an empty config. 2024-09-21T09:53:36.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:35 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:36.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:53:36] ENGINE Serving on http://:::9283 2024-09-21T09:53:36.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:53:36] ENGINE Bus STARTED 2024-09-21T09:53:36.804 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[29191]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:36.804 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[29191]: Standby manager daemon y restarted 2024-09-21T09:53:36.804 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[29191]: Standby manager daemon y started 2024-09-21T09:53:36.804 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:53:36.804 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:36.805 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:53:36.805 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:36.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[40061]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:36.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[40061]: Standby manager daemon y restarted 2024-09-21T09:53:36.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[40061]: Standby manager daemon y started 2024-09-21T09:53:36.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:53:36.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:36.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:53:36.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:36 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:37.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:36 smithi045 ceph-mon[33087]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:37.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:36 smithi045 ceph-mon[33087]: Standby manager daemon y restarted 2024-09-21T09:53:37.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:36 smithi045 ceph-mon[33087]: Standby manager daemon y started 2024-09-21T09:53:37.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:36 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:53:37.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:36 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:37.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:36 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:53:37.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:36 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:37 smithi032 ceph-mon[29191]: mgrmap e27: x(active, since 27s), standbys: y 2024-09-21T09:53:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:37 smithi032 ceph-mon[29191]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:37 smithi032 ceph-mon[40061]: mgrmap e27: x(active, since 27s), standbys: y 2024-09-21T09:53:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:37 smithi032 ceph-mon[40061]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:37 smithi045 ceph-mon[33087]: mgrmap e27: x(active, since 27s), standbys: y 2024-09-21T09:53:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:37 smithi045 ceph-mon[33087]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:38.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:53:38] ENGINE Bus STOPPING 2024-09-21T09:53:38.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:53:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:53:38.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:53:38] ENGINE Bus STOPPED 2024-09-21T09:53:39.034 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:38.541+0000 7fdd40ba1640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-21T09:53:39.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: ignoring --setuser ceph since I am not root 2024-09-21T09:53:39.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: ignoring --setgroup ceph since I am not root 2024-09-21T09:53:39.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:38 smithi045 ceph-mgr[65990]: -- 172.21.15.45:0/641872221 <== mon.1 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d77489b0e0 con 0x55d774879400 2024-09-21T09:53:39.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.155+0000 7f26ccb69640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.155+0000 7f26ccb69640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.155+0000 7f26ccb69640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.155+0000 7f26ccb69640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.155+0000 7f26ccb69640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.156+0000 7f26cfb6f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.156+0000 7f26cfb6f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.156+0000 7f26cfb6f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.434 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.156+0000 7f26cfb6f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.435 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:39.156+0000 7f26cfb6f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:39.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:53:39] ENGINE Bus STARTING 2024-09-21T09:53:39.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:53:39] ENGINE Serving on http://:::9283 2024-09-21T09:53:39.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:53:39] ENGINE Bus STARTED 2024-09-21T09:53:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: Activating manager daemon y 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25159 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: mgrmap e28: y(active, starting, since 0.00661503s) 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:53:39.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: Manager daemon y is now available 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:39.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:39.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: Activating manager daemon y 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25159 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: mgrmap e28: y(active, starting, since 0.00661503s) 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:53:39.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: Manager daemon y is now available 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:39.712 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25159 172.21.15.45:0/1060968995' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: Activating manager daemon y 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25159 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: mgrmap e28: y(active, starting, since 0.00661503s) 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:53:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:53:39.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:53:39.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:53:39.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: Manager daemon y is now available 2024-09-21T09:53:39.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:39.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:39.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:39.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:39.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:40.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:40 smithi045 ceph-mon[33087]: mgrmap e29: y(active, since 1.00937s) 2024-09-21T09:53:40.829 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:40 smithi032 ceph-mon[40061]: mgrmap e29: y(active, since 1.00937s) 2024-09-21T09:53:40.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:40 smithi032 ceph-mon[29191]: mgrmap e29: y(active, since 1.00937s) 2024-09-21T09:53:41.544 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:41.256+0000 7f26af0f3640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:53:41] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: Traceback (most recent call last): 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: self._connections.run(self.expiration_interval) 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: self._run(expiration_interval) 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: s = self.context.wrap_socket( 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:41.545 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: return self.sslsocket_class._create( 2024-09-21T09:53:41.546 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:41.546 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: self.do_handshake() 2024-09-21T09:53:41.546 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:41.546 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: self._sslobj.do_handshake() 2024-09-21T09:53:41.546 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:41.546 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:53:41.856 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-mon[40061]: mgrmap e30: y(active, since 2s) 2024-09-21T09:53:41.856 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:41 smithi032 ceph-mon[29191]: mgrmap e30: y(active, since 2s) 2024-09-21T09:53:41.861 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:41 smithi045 ceph-mon[33087]: mgrmap e30: y(active, since 2s) 2024-09-21T09:53:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:43 smithi045 ceph-mon[33087]: mgrmap e31: y(active, since 4s) 2024-09-21T09:53:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:43 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:43 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:43.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:43 smithi032 ceph-mon[29191]: mgrmap e31: y(active, since 4s) 2024-09-21T09:53:43.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:43 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:43.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:43 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:43.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:43 smithi032 ceph-mon[40061]: mgrmap e31: y(active, since 4s) 2024-09-21T09:53:43.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:43 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:43.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:43 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:53:44] ENGINE Bus STARTING 2024-09-21T09:53:44.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: CherryPy Checker: 2024-09-21T09:53:44.535 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: The Application mounted at '' has an empty config. 2024-09-21T09:53:44.536 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: 2024-09-21T09:53:44.536 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:53:44] ENGINE Serving on http://:::9283 2024-09-21T09:53:44.536 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: [21/Sep/2024:09:53:44] ENGINE Bus STARTED 2024-09-21T09:53:44.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: Standby manager daemon x started 2024-09-21T09:53:44.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: Standby manager daemon x started 2024-09-21T09:53:44.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:53:44.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:44.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:53:44.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:44.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:44.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:44 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:45.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: Standby manager daemon x started 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/144975169' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:44 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:41] ENGINE Bus STARTING 2024-09-21T09:53:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:41] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:53:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:41] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:53:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:53:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout: self.do_handshake() 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:41] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[29191]: [21/Sep/2024:09:53:41] ENGINE Bus STARTED 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[29191]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[29191]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[29191]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:41] ENGINE Bus STARTING 2024-09-21T09:53:45.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:41] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:41] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:53:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout: self.do_handshake() 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:41] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[40061]: [21/Sep/2024:09:53:41] ENGINE Bus STARTED 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[40061]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[40061]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[40061]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-21T09:53:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:45 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:53:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:45 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:41] ENGINE Bus STARTING 2024-09-21T09:53:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:45 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:41] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:53:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:45 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:41] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout: Traceback (most recent call last): 2024-09-21T09:53:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:53:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._run(expiration_interval) 2024-09-21T09:53:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: s = self.context.wrap_socket( 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: return self.sslsocket_class._create( 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: self.do_handshake() 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._sslobj.do_handshake() 2024-09-21T09:53:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:45 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:41] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:53:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:45 smithi045 ceph-mon[33087]: [21/Sep/2024:09:53:41] ENGINE Bus STARTED 2024-09-21T09:53:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:45 smithi045 ceph-mon[33087]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:45 smithi045 ceph-mon[33087]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:53:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:45 smithi045 ceph-mon[33087]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-21T09:53:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:45 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:53:48.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:53:48.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:48.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:48.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:48.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:48.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.265 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T09:53:48.265 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.265 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:48.810 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:53:49.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:48 smithi032 ceph-mon[40061]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:49.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:48 smithi032 ceph-mon[40061]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:48 smithi032 ceph-mon[40061]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:53:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:48 smithi032 ceph-mon[40061]: Reconfiguring daemon prometheus.a on smithi045 2024-09-21T09:53:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:48 smithi032 ceph-mon[29191]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:48 smithi032 ceph-mon[29191]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:48 smithi032 ceph-mon[29191]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:53:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:48 smithi032 ceph-mon[29191]: Reconfiguring daemon prometheus.a on smithi045 2024-09-21T09:53:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:48 smithi045 ceph-mon[33087]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:48 smithi045 ceph-mon[33087]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:53:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:48 smithi045 ceph-mon[33087]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:53:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:48 smithi045 ceph-mon[33087]: Reconfiguring daemon prometheus.a on smithi045 2024-09-21T09:53:49.398 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:53:49.398 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (3m) 6s ago 4m 28.4M - 0.25.0 c8568f914cd2 75fcd3b3b5dd 2024-09-21T09:53:49.398 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (4m) 5s ago 4m 89.3M - 9.4.7 954c08fa6188 419a19b40991 2024-09-21T09:53:49.398 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (3m) 6s ago 3m 16.2M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (3m) 5s ago 3m 16.4M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (60s) 5s ago 8m 123M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5cc590ae518b 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (19s) 6s ago 10m 489M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (10m) 6s ago 10m 55.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 017caecda0d9 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (8m) 5s ago 8m 48.2M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (8m) 6s ago 8m 44.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 125675da4b84 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (4m) 6s ago 3m 16.8M - 1.5.0 0da6a335fe13 e4bf1511c584 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (3m) 5s ago 3m 15.2M - 1.5.0 0da6a335fe13 49083320e62a 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (8m) 6s ago 8m 57.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (7m) 6s ago 7m 51.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (7m) 6s ago 7m 57.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (6m) 6s ago 6m 44.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (6m) 5s ago 6m 52.6M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (6m) 5s ago 6m 47.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:53:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (5m) 5s ago 5m 46.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:53:49.400 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (5m) 5s ago 5m 49.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:53:49.400 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (24s) 5s ago 4m 36.2M - 2.43.0 a07b618ecd1d 6a0f34cf3495 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:53:49.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:53:49.761 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:53:49.761 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:53:49.761 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:53:49.761 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:53:49.761 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:53:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:49 smithi045 ceph-mon[33087]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-21T09:53:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:49 smithi045 ceph-mon[33087]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:49 smithi045 ceph-mon[33087]: from='client.25571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:49 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/1031570691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [ 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr" 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "", 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:53:50.064 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:53:50.065 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:49 smithi032 ceph-mon[40061]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-21T09:53:50.065 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:49 smithi032 ceph-mon[40061]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:50.065 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:49 smithi032 ceph-mon[40061]: from='client.25571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:50.065 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:49 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/1031570691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:50.065 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:49 smithi032 ceph-mon[29191]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-21T09:53:50.065 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:49 smithi032 ceph-mon[29191]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:50.065 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:49 smithi032 ceph-mon[29191]: from='client.25571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:50.065 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:49 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/1031570691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:50.898 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[65986]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:53:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:53:51.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:50 smithi032 ceph-mon[29191]: from='client.25577 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:50 smithi032 ceph-mon[29191]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:50 smithi032 ceph-mon[40061]: from='client.25577 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:50 smithi032 ceph-mon[40061]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:50 smithi045 ceph-mon[33087]: from='client.25577 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:50 smithi045 ceph-mon[33087]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:51.285 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 systemd[1]: Stopping Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:51.570 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.292Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:53:51.570 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.292Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:53:51.570 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.292Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:53:51.570 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.292Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:53:51.571 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.292Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:53:51.571 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.292Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:53:51.571 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.292Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:53:51.571 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.292Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:53:51.571 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.293Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:53:51.571 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.293Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:53:51.571 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.293Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:53:51.571 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[73745]: ts=2024-09-21T09:53:51.293Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:53:51.571 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 podman[75614]: 2024-09-21 09:53:51.302980894 +0000 UTC m=+0.116065105 container died 6a0f34cf349510247fc9e958083fde8230cc8a45e53434a3ea0ee0bf79ba9da3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:51.949 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 podman[75614]: 2024-09-21 09:53:51.750671211 +0000 UTC m=+0.563755422 container cleanup 6a0f34cf349510247fc9e958083fde8230cc8a45e53434a3ea0ee0bf79ba9da3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:51.950 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:51 smithi045 bash[75614]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:53:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:51 smithi032 ceph-mon[29191]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-21T09:53:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:51 smithi032 ceph-mon[40061]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-21T09:53:52.213 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 podman[75626]: 2024-09-21 09:53:52.052257092 +0000 UTC m=+0.751759735 container remove 6a0f34cf349510247fc9e958083fde8230cc8a45e53434a3ea0ee0bf79ba9da3 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:52.214 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:51 smithi045 ceph-mon[33087]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-21T09:53:52.535 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:53:52.535 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 systemd[1]: Stopped Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:52.535 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.055s CPU time. 2024-09-21T09:53:52.535 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 systemd[1]: Starting Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:52.535 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 podman[75721]: 2024-09-21 09:53:52.347151976 +0000 UTC m=+0.017760359 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:53:52.535 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 podman[75721]: 2024-09-21 09:53:52.461607845 +0000 UTC m=+0.132216218 container create 9e8fcaa2a6a3eaba245b441a33f6dcd1f01d849d4f888fde9b2860b905ca038f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:53.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 podman[75721]: 2024-09-21 09:53:52.680556044 +0000 UTC m=+0.351164414 container init 9e8fcaa2a6a3eaba245b441a33f6dcd1f01d849d4f888fde9b2860b905ca038f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:53.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 podman[75721]: 2024-09-21 09:53:52.683357466 +0000 UTC m=+0.353965838 container start 9e8fcaa2a6a3eaba245b441a33f6dcd1f01d849d4f888fde9b2860b905ca038f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:53.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.708Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:53:53.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.708Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.708Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi045 (none))" 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.708Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.708Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.710Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.711Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.711Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.711Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.714Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.714Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.458µs 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.714Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:53:53.036 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.715Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.724Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.725Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.725Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.725Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.477µs wal_replay_duration=10.77629ms wbl_replay_duration=89ns total_replay_duration=10.850304ms 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.726Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.726Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.726Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.748Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.239257ms db_storage=884ns remote_storage=1.705µs web_handler=379ns query_engine=568ns scrape=458.655µs scrape_sd=164.84µs notify=12.325µs notify_sd=10.586µs rules=20.204129ms tracing=5.969µs 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.748Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:53:52.748Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 bash[75721]: 9e8fcaa2a6a3eaba245b441a33f6dcd1f01d849d4f888fde9b2860b905ca038f 2024-09-21T09:53:53.037 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:53:52 smithi045 systemd[1]: Started Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:54.130 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:53 smithi045 ceph-mon[33087]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:53:54.131 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:53 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:54.131 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:53 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:54.131 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:53 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:54.131 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:53 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:54.131 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:53 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[40061]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[29191]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:54.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:54.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:53 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[29191]: Upgrade: Updating mgr.x 2024-09-21T09:53:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:53:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[29191]: Deploying daemon mgr.x on smithi045 2024-09-21T09:53:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[40061]: Upgrade: Updating mgr.x 2024-09-21T09:53:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:53:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:55.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:54 smithi032 ceph-mon[40061]: Deploying daemon mgr.x on smithi045 2024-09-21T09:53:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:54 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:54 smithi045 ceph-mon[33087]: Upgrade: Updating mgr.x 2024-09-21T09:53:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:54 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:54 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:53:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:54 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:55.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:54 smithi045 ceph-mon[33087]: Deploying daemon mgr.x on smithi045 2024-09-21T09:53:56.175 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:55 smithi045 ceph-mon[33087]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:53:56.175 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:55 smithi045 systemd[1]: Stopping Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:56.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:55 smithi032 ceph-mon[29191]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:53:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:55 smithi032 ceph-mon[40061]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:53:56.451 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:56 smithi045 podman[75994]: 2024-09-21 09:53:56.173555425 +0000 UTC m=+0.151164964 container died 5cc590ae518b644bf411dd4810551e71e5776efd6c1e385a0347b9c0a1efb39a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T09:53:56.785 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:56 smithi045 podman[75994]: 2024-09-21 09:53:56.616245515 +0000 UTC m=+0.593855082 container cleanup 5cc590ae518b644bf411dd4810551e71e5776efd6c1e385a0347b9c0a1efb39a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b) 2024-09-21T09:53:56.785 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:56 smithi045 bash[75994]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:53:57.066 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:56 smithi045 podman[76008]: 2024-09-21 09:53:56.80918255 +0000 UTC m=+0.633082817 container remove 5cc590ae518b644bf411dd4810551e71e5776efd6c1e385a0347b9c0a1efb39a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, ceph=True, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-21T09:53:57.066 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:56 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:53:57.066 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:53:57.066 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 systemd[1]: Stopped Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:57.066 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x.service: Consumed 18.052s CPU time. 2024-09-21T09:53:57.473 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 systemd[1]: Starting Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:57.731 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 podman[76133]: 2024-09-21 09:53:57.375200709 +0000 UTC m=+0.016281052 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:53:57.731 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 podman[76133]: 2024-09-21 09:53:57.506631853 +0000 UTC m=+0.147712184 container create 7ecdc05240855f70b1581bbc7751e684bd0be5e6f3002f472736f315f0f52592 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T09:53:57.731 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 podman[76133]: 2024-09-21 09:53:57.729878695 +0000 UTC m=+0.370959033 container init 7ecdc05240855f70b1581bbc7751e684bd0be5e6f3002f472736f315f0f52592 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-21T09:53:58.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:57 smithi045 ceph-mon[33087]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:53:58.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 podman[76133]: 2024-09-21 09:53:57.732881795 +0000 UTC m=+0.373962127 container start 7ecdc05240855f70b1581bbc7751e684bd0be5e6f3002f472736f315f0f52592 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T09:53:58.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 ceph-mgr[76153]: -- 172.21.15.45:0/446506746 <== mon.1 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ad5a59d0e0 con 0x55ad5a57a800 2024-09-21T09:53:58.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 bash[76133]: 7ecdc05240855f70b1581bbc7751e684bd0be5e6f3002f472736f315f0f52592 2024-09-21T09:53:58.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:53:57 smithi045 systemd[1]: Started Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:58.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:57 smithi032 ceph-mon[29191]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:53:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:57 smithi032 ceph-mon[40061]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:53:59.197 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:58 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:59.197 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:58 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:59.197 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:58 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:58 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:58 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:58 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:59.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:58 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:59.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:58 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:53:59.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:58 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:00.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:53:59 smithi032 ceph-mon[29191]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:53:59 smithi032 ceph-mon[40061]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:00.223 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:53:59 smithi045 ceph-mon[33087]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:01 smithi032 ceph-mon[29191]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:01 smithi032 ceph-mon[40061]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:02.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:01 smithi045 ceph-mon[33087]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:03.388 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.388 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.388 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.388 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.388 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: [21/Sep/2024:09:54:03] ENGINE Bus STARTING 2024-09-21T09:54:03.389 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: CherryPy Checker: 2024-09-21T09:54:03.389 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: The Application mounted at '' has an empty config. 2024-09-21T09:54:03.389 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: 2024-09-21T09:54:03.389 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: [21/Sep/2024:09:54:03] ENGINE Serving on http://:::9283 2024-09-21T09:54:03.389 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:54:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: [21/Sep/2024:09:54:03] ENGINE Bus STARTED 2024-09-21T09:54:03.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:03 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:03 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:03 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:03 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:03 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:03 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:03 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:03 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:03.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:54:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:54:03] "GET /metrics HTTP/1.1" 200 33806 "" "Prometheus/2.43.0" 2024-09-21T09:54:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[29191]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[29191]: Standby manager daemon x restarted 2024-09-21T09:54:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[29191]: Standby manager daemon x started 2024-09-21T09:54:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:54:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:54:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:54:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[29191]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:54:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[40061]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[40061]: Standby manager daemon x restarted 2024-09-21T09:54:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[40061]: Standby manager daemon x started 2024-09-21T09:54:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:54:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:54:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:54:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:04 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:54:04.478 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:04 smithi045 ceph-mon[33087]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:04.478 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:04 smithi045 ceph-mon[33087]: Standby manager daemon x restarted 2024-09-21T09:54:04.478 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:04 smithi045 ceph-mon[33087]: Standby manager daemon x started 2024-09-21T09:54:04.478 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:04 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:54:04.478 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:04 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:54:04.478 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:04 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:54:04.479 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:04 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/1971348216' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:54:05.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:05 smithi032 ceph-mon[29191]: mgrmap e33: y(active, since 25s), standbys: x 2024-09-21T09:54:05.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:05 smithi032 ceph-mon[40061]: mgrmap e33: y(active, since 25s), standbys: x 2024-09-21T09:54:05.508 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:05 smithi045 ceph-mon[33087]: mgrmap e33: y(active, since 25s), standbys: x 2024-09-21T09:54:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.440 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:54:06.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.443 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.445 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.445 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:06.445 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:06.445 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all mgr 2024-09-21T09:54:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all crash 2024-09-21T09:54:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all rgw 2024-09-21T09:54:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:54:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T09:54:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:54:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:54:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all nfs 2024-09-21T09:54:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:54:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all mgr 2024-09-21T09:54:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all crash 2024-09-21T09:54:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all rgw 2024-09-21T09:54:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:54:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T09:54:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:54:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:54:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all nfs 2024-09-21T09:54:07.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:07 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:54:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:07 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all mgr 2024-09-21T09:54:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:07 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all crash 2024-09-21T09:54:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:07 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all rgw 2024-09-21T09:54:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:07 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:54:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:07 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T09:54:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:07 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:54:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:07 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:54:07.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:07 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all nfs 2024-09-21T09:54:07.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:07 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:54:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:08 smithi032 ceph-mon[29191]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:08 smithi032 ceph-mon[40061]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:08.207 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:08 smithi032 systemd[1]: Stopping Ceph node-exporter.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:08.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:08 smithi045 ceph-mon[33087]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:08.667 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:08 smithi032 podman[92457]: 2024-09-21 09:54:08.408575261 +0000 UTC m=+0.134371998 container died e4bf1511c58419374805070c66e12ee96d49d4f1bd25c314dcbde3bcab9bd26f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:08.957 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:08 smithi032 podman[92457]: 2024-09-21 09:54:08.755840468 +0000 UTC m=+0.481637192 container cleanup e4bf1511c58419374805070c66e12ee96d49d4f1bd25c314dcbde3bcab9bd26f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:08.957 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:08 smithi032 bash[92457]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T09:54:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:09 smithi032 ceph-mon[40061]: Upgrade: Updating node-exporter.a (1/2) 2024-09-21T09:54:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:09 smithi032 ceph-mon[40061]: Deploying daemon node-exporter.a on smithi032 2024-09-21T09:54:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:09 smithi032 ceph-mon[29191]: Upgrade: Updating node-exporter.a (1/2) 2024-09-21T09:54:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:09 smithi032 ceph-mon[29191]: Deploying daemon node-exporter.a on smithi032 2024-09-21T09:54:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:09.457 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 podman[92469]: 2024-09-21 09:54:09.107095258 +0000 UTC m=+0.697707127 container remove e4bf1511c58419374805070c66e12ee96d49d4f1bd25c314dcbde3bcab9bd26f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:09.458 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:54:09.458 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T09:54:09.458 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 systemd[1]: Stopped Ceph node-exporter.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:09.458 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 2.238s CPU time. 2024-09-21T09:54:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:09 smithi045 ceph-mon[33087]: Upgrade: Updating node-exporter.a (1/2) 2024-09-21T09:54:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:09 smithi045 ceph-mon[33087]: Deploying daemon node-exporter.a on smithi032 2024-09-21T09:54:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:09.875 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 systemd[1]: Starting Ceph node-exporter.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:09.875 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 bash[92598]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:54:10.151 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 bash[92598]: Getting image source signatures 2024-09-21T09:54:10.152 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 bash[92598]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:54:10.152 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 bash[92598]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:54:10.152 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:09 smithi032 bash[92598]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:54:10.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:10 smithi032 ceph-mon[29191]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:10.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:10 smithi032 ceph-mon[40061]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:10.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:10 smithi045 ceph-mon[33087]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:11.888 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:11 smithi032 bash[92598]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:54:11.888 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:11 smithi032 bash[92598]: Writing manifest to image destination 2024-09-21T09:54:12.154 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:11 smithi032 podman[92598]: 2024-09-21 09:54:11.858729744 +0000 UTC m=+2.293059112 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:54:12.154 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:11 smithi032 podman[92598]: 2024-09-21 09:54:11.95700239 +0000 UTC m=+2.391331745 container create e54d1ecd8cb39af5350faa58eec1ea4053cc2643d4dec51523579ba1df8feb6f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:12.445 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-mon[40061]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:12.445 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-mon[29191]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:12.446 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 podman[92598]: 2024-09-21 09:54:12.192406245 +0000 UTC m=+2.626735607 container init e54d1ecd8cb39af5350faa58eec1ea4053cc2643d4dec51523579ba1df8feb6f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:12.446 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 podman[92598]: 2024-09-21 09:54:12.196071727 +0000 UTC m=+2.630401084 container start e54d1ecd8cb39af5350faa58eec1ea4053cc2643d4dec51523579ba1df8feb6f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:12.446 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.199Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:54:12.446 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.199Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:54:12.446 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.200Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:54:12.446 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.200Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:54:12.446 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:54:12.447 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:54:12.448 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:54:12.449 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:54:12.450 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:54:12.450 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.201Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:54:12.450 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.202Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:54:12.450 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a[92670]: ts=2024-09-21T09:54:12.202Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:54:12.450 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 bash[92598]: e54d1ecd8cb39af5350faa58eec1ea4053cc2643d4dec51523579ba1df8feb6f 2024-09-21T09:54:12.450 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 09:54:12 smithi032 systemd[1]: Started Ceph node-exporter.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:12.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:12 smithi045 ceph-mon[33087]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:13.624 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:13 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:13.624 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:13 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:13.624 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:13 smithi032 ceph-mon[40061]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:13.624 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:13 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:13.624 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:13 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:13.625 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:13 smithi032 ceph-mon[29191]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:13 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:13.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:13 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:13.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:13 smithi045 ceph-mon[33087]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:13.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:54:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:54:13] "GET /metrics HTTP/1.1" 200 33804 "" "Prometheus/2.43.0" 2024-09-21T09:54:14.284 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:14 smithi045 systemd[1]: Stopping Ceph node-exporter.b for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:14.682 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:14 smithi045 podman[79463]: 2024-09-21 09:54:14.406749673 +0000 UTC m=+0.158368930 container died 49083320e62a4b2a69642b805b2f7ee97c26752e10c59890dd131991c3378043 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:54:14.682 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:14 smithi045 ceph-mon[33087]: Upgrade: Updating node-exporter.b (2/2) 2024-09-21T09:54:14.682 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:14 smithi045 ceph-mon[33087]: Deploying daemon node-exporter.b on smithi045 2024-09-21T09:54:14.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:14 smithi032 ceph-mon[29191]: Upgrade: Updating node-exporter.b (2/2) 2024-09-21T09:54:14.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:14 smithi032 ceph-mon[29191]: Deploying daemon node-exporter.b on smithi045 2024-09-21T09:54:14.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:14 smithi032 ceph-mon[40061]: Upgrade: Updating node-exporter.b (2/2) 2024-09-21T09:54:14.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:14 smithi032 ceph-mon[40061]: Deploying daemon node-exporter.b on smithi045 2024-09-21T09:54:15.015 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:14 smithi045 podman[79463]: 2024-09-21 09:54:14.779566071 +0000 UTC m=+0.531185315 container cleanup 49083320e62a4b2a69642b805b2f7ee97c26752e10c59890dd131991c3378043 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:54:15.015 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:14 smithi045 bash[79463]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T09:54:15.284 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 podman[79475]: 2024-09-21 09:54:15.014166081 +0000 UTC m=+0.610432965 container remove 49083320e62a4b2a69642b805b2f7ee97c26752e10c59890dd131991c3378043 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:54:15.285 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:54:15.285 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T09:54:15.285 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 systemd[1]: Stopped Ceph node-exporter.b for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:15.285 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 2.222s CPU time. 2024-09-21T09:54:15.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:15 smithi032 ceph-mon[29191]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:15.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:15 smithi032 ceph-mon[40061]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:15.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:15 smithi045 ceph-mon[33087]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:15.785 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 systemd[1]: Starting Ceph node-exporter.b for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:15.785 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 bash[79605]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:54:16.285 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 bash[79605]: Getting image source signatures 2024-09-21T09:54:16.285 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 bash[79605]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:54:16.285 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 bash[79605]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:54:16.285 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:15 smithi045 bash[79605]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:54:17.785 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:17 smithi045 bash[79605]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:54:17.785 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:17 smithi045 bash[79605]: Writing manifest to image destination 2024-09-21T09:54:17.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:17 smithi045 ceph-mon[33087]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:17.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:17 smithi032 ceph-mon[29191]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:17 smithi032 ceph-mon[40061]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:18.093 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:17 smithi045 podman[79605]: 2024-09-21 09:54:17.791822464 +0000 UTC m=+2.315566960 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:54:18.094 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:17 smithi045 podman[79605]: 2024-09-21 09:54:17.890240657 +0000 UTC m=+2.413985154 container create 61f6c7aa590c5a5d99eeaa09c5df2c4065bc43be8da20c7759e537f23a0c4c7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:54:18.535 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 podman[79605]: 2024-09-21 09:54:18.092510511 +0000 UTC m=+2.616255010 container init 61f6c7aa590c5a5d99eeaa09c5df2c4065bc43be8da20c7759e537f23a0c4c7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:54:18.535 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 podman[79605]: 2024-09-21 09:54:18.096046133 +0000 UTC m=+2.619790630 container start 61f6c7aa590c5a5d99eeaa09c5df2c4065bc43be8da20c7759e537f23a0c4c7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:54:18.535 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.099Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:54:18.535 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.099Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.100Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.100Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.100Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.100Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:54:18.536 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:54:18.537 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:54:18.537 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:54:18.537 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:54:18.537 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:54:18.537 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:54:18.537 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:54:18.537 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:54:18.538 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:54:18.539 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.101Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.102Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b[79679]: ts=2024-09-21T09:54:18.102Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 bash[79605]: 61f6c7aa590c5a5d99eeaa09c5df2c4065bc43be8da20c7759e537f23a0c4c7c 2024-09-21T09:54:18.540 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 09:54:18 smithi045 systemd[1]: Started Ceph node-exporter.b for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:19.526 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:19 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:19.526 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:19 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:19.526 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:19 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:19.586 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:19 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:19.586 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:19 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:19.586 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:19 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:19.586 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:19 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:19.586 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:19 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:19.587 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:19 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:20.399 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:54:20.516 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:20 smithi032 ceph-mon[29191]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:20.516 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:20 smithi032 ceph-mon[40061]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:20.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:20 smithi045 ceph-mon[33087]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:20.997 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:54:20.997 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (4m) 37s ago 5m 28.4M - 0.25.0 c8568f914cd2 75fcd3b3b5dd 2024-09-21T09:54:20.997 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (4m) 18s ago 4m 90.7M - 9.4.7 954c08fa6188 419a19b40991 2024-09-21T09:54:20.997 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (3m) 37s ago 3m 16.2M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:54:20.997 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (3m) 18s ago 3m 16.7M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:54:20.997 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (23s) 18s ago 8m 66.0M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T09:54:20.997 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (50s) 37s ago 10m 489M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T09:54:20.997 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (10m) 37s ago 10m 55.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 017caecda0d9 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (9m) 18s ago 9m 49.2M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (9m) 37s ago 9m 44.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 125675da4b84 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 starting - - - - 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 starting - - - - 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (8m) 37s ago 8m 57.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (8m) 37s ago 8m 51.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (7m) 37s ago 7m 57.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (7m) 37s ago 7m 44.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (7m) 18s ago 7m 52.9M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (6m) 18s ago 6m 47.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (6m) 18s ago 6m 46.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (5m) 18s ago 5m 49.3M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:54:20.998 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (28s) 18s ago 5m 38.4M - 2.43.0 a07b618ecd1d 9e8fcaa2a6a3 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:54:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:54:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:54:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:54:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:54:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:54:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:54:21.367 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [ 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr" 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading node-exporter daemons", 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:54:21.668 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:54:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:22 smithi045 ceph-mon[33087]: from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:22 smithi045 ceph-mon[33087]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:22 smithi045 ceph-mon[33087]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:22 smithi045 ceph-mon[33087]: from='client.25610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:22 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/624423330' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:22 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:22 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[29191]: from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[29191]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[29191]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[29191]: from='client.25610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/624423330' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:22.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:22.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[40061]: from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:22.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[40061]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:22.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[40061]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:22.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[40061]: from='client.25610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:22.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/624423330' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:22.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:22.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:22 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[40061]: from='client.25622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[29191]: from='client.25622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:23 smithi045 ceph-mon[33087]: from='client.25622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:23 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:23 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:23 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:23 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:23 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:23 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:54:23 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:54:23] "GET /metrics HTTP/1.1" 200 33804 "" "Prometheus/2.43.0" 2024-09-21T09:54:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:24 smithi032 ceph-mon[40061]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:24 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:24 smithi032 ceph-mon[29191]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:24 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:24.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:24 smithi045 ceph-mon[33087]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:24.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:24 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:26.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:26.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:26.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:26.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:26.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:28.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:28 smithi045 ceph-mon[33087]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:28.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:28 smithi032 ceph-mon[29191]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:28.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:28 smithi032 ceph-mon[40061]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:29.608 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:29 smithi032 ceph-mon[29191]: Upgrade: Updating prometheus.a 2024-09-21T09:54:29.608 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:29 smithi032 ceph-mon[29191]: Deploying daemon prometheus.a on smithi045 2024-09-21T09:54:29.608 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:29 smithi032 ceph-mon[40061]: Upgrade: Updating prometheus.a 2024-09-21T09:54:29.609 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:29 smithi032 ceph-mon[40061]: Deploying daemon prometheus.a on smithi045 2024-09-21T09:54:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:29 smithi045 ceph-mon[33087]: Upgrade: Updating prometheus.a 2024-09-21T09:54:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:29 smithi045 ceph-mon[33087]: Deploying daemon prometheus.a on smithi045 2024-09-21T09:54:30.632 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:30 smithi032 ceph-mon[40061]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:30.632 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:30 smithi032 ceph-mon[29191]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:30.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:30 smithi045 ceph-mon[33087]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:32.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:32 smithi032 ceph-mon[29191]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:32.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:32 smithi032 ceph-mon[40061]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:32.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:32 smithi045 ceph-mon[33087]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:33.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:54:33 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:54:33] "GET /metrics HTTP/1.1" 200 33801 "" "Prometheus/2.43.0" 2024-09-21T09:54:34.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:34 smithi032 ceph-mon[29191]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:34.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:34 smithi032 ceph-mon[40061]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:34.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:34 smithi045 ceph-mon[33087]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:36.429 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:36 smithi045 ceph-mon[33087]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:36.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:36 smithi032 ceph-mon[29191]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:36.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:36 smithi032 ceph-mon[40061]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:38.651 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:38 smithi045 ceph-mon[33087]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:38.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:38 smithi032 ceph-mon[29191]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:38.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:38 smithi032 ceph-mon[40061]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:39.422 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:39 smithi045 ceph-mon[33087]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:39.422 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:39 smithi032 ceph-mon[29191]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:39.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:39 smithi032 ceph-mon[40061]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:39.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:39 smithi045 systemd[1]: Stopping Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.042Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.042Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.042Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.042Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.042Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.042Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.042Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.042Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:54:40.286 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.042Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:54:40.287 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.043Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:54:40.287 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.043Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:54:40.287 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[75737]: ts=2024-09-21T09:54:40.043Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:54:40.287 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 podman[83664]: 2024-09-21 09:54:40.052533301 +0000 UTC m=+0.097809320 container died 9e8fcaa2a6a3eaba245b441a33f6dcd1f01d849d4f888fde9b2860b905ca038f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:40.594 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 podman[83664]: 2024-09-21 09:54:40.374750914 +0000 UTC m=+0.420026920 container cleanup 9e8fcaa2a6a3eaba245b441a33f6dcd1f01d849d4f888fde9b2860b905ca038f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:40.594 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 bash[83664]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:54:41.024 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 podman[83677]: 2024-09-21 09:54:40.592837001 +0000 UTC m=+0.540783092 container remove 9e8fcaa2a6a3eaba245b441a33f6dcd1f01d849d4f888fde9b2860b905ca038f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:41.025 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:54:41.025 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 systemd[1]: Stopped Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:41.025 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:40 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.121s CPU time. 2024-09-21T09:54:41.285 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 systemd[1]: Starting Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:41.285 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 podman[83802]: 2024-09-21 09:54:41.136693972 +0000 UTC m=+0.017144379 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-21T09:54:41.285 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 podman[83802]: 2024-09-21 09:54:41.260519684 +0000 UTC m=+0.140970124 container create 2d9f1cee2e82c4dce50997db04bdf78f6c43d60b0a52ff8c69a4db1fbb8954a3 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:41.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:41 smithi032 ceph-mon[29191]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:41 smithi032 ceph-mon[40061]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-mon[33087]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:41.785 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 podman[83802]: 2024-09-21 09:54:41.487895713 +0000 UTC m=+0.368346120 container init 2d9f1cee2e82c4dce50997db04bdf78f6c43d60b0a52ff8c69a4db1fbb8954a3 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:41.785 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 podman[83802]: 2024-09-21 09:54:41.490632764 +0000 UTC m=+0.371083168 container start 2d9f1cee2e82c4dce50997db04bdf78f6c43d60b0a52ff8c69a4db1fbb8954a3 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:41.785 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.511Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-21T09:54:41.785 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.511Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-21T09:54:41.785 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.511Z caller=main.go:623 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi045 (none))" 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.511Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.511Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.513Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.513Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.515Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.515Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.516Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.517Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.692µs 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.517Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.518Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.524Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.524Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-21T09:54:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.525Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.526Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.526Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=41.286µs wal_replay_duration=8.993359ms wbl_replay_duration=110ns total_replay_duration=9.066596ms 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.528Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.528Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.528Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.544Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.904553ms db_storage=829ns remote_storage=1.163µs web_handler=301ns query_engine=615ns scrape=437.943µs scrape_sd=108.432µs notify=10.84µs notify_sd=13.306µs rules=14.963777ms tracing=5.938µs 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.544Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T09:54:41.544Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 bash[83802]: 2d9f1cee2e82c4dce50997db04bdf78f6c43d60b0a52ff8c69a4db1fbb8954a3 2024-09-21T09:54:41.787 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 09:54:41 smithi045 systemd[1]: Started Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:42.900 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:42 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.900 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:42 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.900 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:42 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:42.901 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:42 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.901 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:42 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:42.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:42 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:44.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:43 smithi032 ceph-mon[29191]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:43 smithi032 ceph-mon[40061]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:44.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:43 smithi045 ceph-mon[33087]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:45 smithi032 ceph-mon[29191]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:45 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:45 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:45 smithi032 ceph-mon[40061]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:45 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:45 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:46.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:45 smithi045 ceph-mon[33087]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:45 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:45 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:47.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:47.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:47 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:47.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:47.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:47 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:47.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:47.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:47 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:48.446 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:48 smithi045 ceph-mon[33087]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:48.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:48 smithi032 ceph-mon[29191]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:48.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:48 smithi032 ceph-mon[40061]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:49.863 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:54:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:54:49] "GET /metrics HTTP/1.1" 200 33802 "" "Prometheus/2.51.0" 2024-09-21T09:54:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:50 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:50.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:50.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:50.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:50.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:50.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:54:50.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:50.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:50 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:51.985 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:54:52.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:52 smithi032 ceph-mon[40061]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:52.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:52 smithi032 ceph-mon[40061]: Upgrade: Updating alertmanager.a 2024-09-21T09:54:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:52 smithi032 ceph-mon[40061]: Deploying daemon alertmanager.a on smithi032 2024-09-21T09:54:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:52 smithi032 ceph-mon[29191]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:52 smithi032 ceph-mon[29191]: Upgrade: Updating alertmanager.a 2024-09-21T09:54:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:52 smithi032 ceph-mon[29191]: Deploying daemon alertmanager.a on smithi032 2024-09-21T09:54:52.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:52 smithi045 ceph-mon[33087]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:52.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:52 smithi045 ceph-mon[33087]: Upgrade: Updating alertmanager.a 2024-09-21T09:54:52.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:52 smithi045 ceph-mon[33087]: Deploying daemon alertmanager.a on smithi032 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (4m) 30s ago 5m 27.1M - 0.25.0 c8568f914cd2 75fcd3b3b5dd 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (5m) 7s ago 5m 88.9M - 9.4.7 954c08fa6188 419a19b40991 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (4m) 30s ago 4m 16.6M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (4m) 7s ago 4m 16.9M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (55s) 7s ago 9m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (82s) 30s ago 11m 508M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (11m) 30s ago 11m 57.2M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 017caecda0d9 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (9m) 7s ago 9m 47.0M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (9m) 30s ago 9m 47.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 125675da4b84 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (40s) 30s ago 5m 7847k - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T09:54:52.662 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (34s) 7s ago 4m 10.2M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T09:54:52.663 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (9m) 30s ago 9m 57.8M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:54:52.663 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (8m) 30s ago 8m 52.1M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:54:52.663 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (8m) 30s ago 8m 57.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:54:52.663 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (7m) 30s ago 7m 44.8M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:54:52.663 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (7m) 7s ago 7m 53.1M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:54:52.663 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (7m) 7s ago 7m 47.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:54:52.663 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (6m) 7s ago 6m 46.8M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:54:52.663 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (6m) 7s ago 6m 49.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:54:52.663 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (11s) 7s ago 5m 32.8M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T09:54:53.032 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:54:53.032 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:54:53.032 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:54:53.032 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:54:53.032 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:54:53.032 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:54:53.032 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:54:53.032 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:54:53.032 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:54:53.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:54:53.033 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:54:53.033 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:54:53.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:54:53.033 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:54:53.033 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:54:53.033 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:54:53.033 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:54:53.033 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:54:53.337 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [ 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr" 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading alertmanager daemons", 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:54:53.338 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:54:53.338 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:53 smithi032 ceph-mon[29191]: from='client.15741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:53.339 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:53 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/712427494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:53.339 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:53 smithi032 ceph-mon[40061]: from='client.15741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:53.339 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:53 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/712427494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:53 smithi045 ceph-mon[33087]: from='client.15741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:53 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/712427494' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:54.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:54 smithi032 ceph-mon[29191]: from='client.25634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:54.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:54 smithi032 ceph-mon[29191]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:54.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:54 smithi032 ceph-mon[29191]: from='client.25640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:54.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:54 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:54 smithi032 ceph-mon[40061]: from='client.25634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:54 smithi032 ceph-mon[40061]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:54 smithi032 ceph-mon[40061]: from='client.25640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:54 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:54.472 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:54 smithi045 ceph-mon[33087]: from='client.25634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:54.473 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:54 smithi045 ceph-mon[33087]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:54.473 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:54 smithi045 ceph-mon[33087]: from='client.25640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:54.473 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:54 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:55.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:55 smithi032 ceph-mon[29191]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:55.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:55 smithi032 ceph-mon[40061]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:55.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:55 smithi045 ceph-mon[33087]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:56.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:56 smithi032 ceph-mon[29191]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:56.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:56 smithi032 ceph-mon[40061]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:56.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:56 smithi045 ceph-mon[33087]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:54:58 smithi032 ceph-mon[40061]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:54:58 smithi032 ceph-mon[29191]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:58.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:54:58 smithi045 ceph-mon[33087]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:54:59.177 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:54:58 smithi032 systemd[1]: Stopping Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:59.177 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:54:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[73246]: ts=2024-09-21T09:54:59.134Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:54:59.436 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:54:59 smithi032 podman[96579]: 2024-09-21 09:54:59.175539207 +0000 UTC m=+0.139011352 container died 75fcd3b3b5dda2756a64e9c0b140c48e554c51bc52923b7c676d4e7aa1e58c92 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:59.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:54:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:54:59] "GET /metrics HTTP/1.1" 200 33798 "" "Prometheus/2.51.0" 2024-09-21T09:54:59.707 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:54:59 smithi032 podman[96579]: 2024-09-21 09:54:59.635323299 +0000 UTC m=+0.598795477 container cleanup 75fcd3b3b5dda2756a64e9c0b140c48e554c51bc52923b7c676d4e7aa1e58c92 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:59.707 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:54:59 smithi032 bash[96579]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:55:00.316 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:00 smithi032 ceph-mon[29191]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:00.317 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:00 smithi032 ceph-mon[40061]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:00.317 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:54:59 smithi032 podman[96593]: 2024-09-21 09:54:59.961492008 +0000 UTC m=+0.777284869 container remove 75fcd3b3b5dda2756a64e9c0b140c48e554c51bc52923b7c676d4e7aa1e58c92 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:55:00.317 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:00 smithi032 podman[96593]: 2024-09-21 09:55:00.147425152 +0000 UTC m=+0.963218013 volume remove 50096254d7bc7ca0b1d3a99bd9e3fee7ccb297456531dc2dd321f9955f51dd04 2024-09-21T09:55:00.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:00 smithi045 ceph-mon[33087]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:00.600 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:00 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:55:00.600 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:00 smithi032 systemd[1]: Stopped Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:55:00.600 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:00 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 1.198s CPU time. 2024-09-21T09:55:00.600 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:00 smithi032 systemd[1]: Starting Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:55:00.850 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:00 smithi032 podman[96720]: 2024-09-21 09:55:00.662479897 +0000 UTC m=+0.016935482 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-21T09:55:00.851 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:00 smithi032 podman[96720]: 2024-09-21 09:55:00.765712976 +0000 UTC m=+0.120168563 volume create 70e01a1d853493fb928a16458a23d9b6741a7bd564a635ee1e3f5f32f0b7ed0a 2024-09-21T09:55:01.147 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:00 smithi032 podman[96720]: 2024-09-21 09:55:00.849431561 +0000 UTC m=+0.203887145 container create d6fa4f2dece853e9a0d502d86f167b2dac9001bedad89290ee0fe27abee67b45 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:55:01.147 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 podman[96720]: 2024-09-21 09:55:01.1439979 +0000 UTC m=+0.498453493 container init d6fa4f2dece853e9a0d502d86f167b2dac9001bedad89290ee0fe27abee67b45 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 podman[96720]: 2024-09-21 09:55:01.148696456 +0000 UTC m=+0.503152041 container start d6fa4f2dece853e9a0d502d86f167b2dac9001bedad89290ee0fe27abee67b45 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:01.161Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:01.161Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:01.162Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.32 port=9094 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:01.164Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:01.185Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:01.186Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:01.187Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:01.187Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:55:01.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 bash[96720]: d6fa4f2dece853e9a0d502d86f167b2dac9001bedad89290ee0fe27abee67b45 2024-09-21T09:55:01.458 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:01 smithi032 systemd[1]: Started Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:55:02.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[40061]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:02.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:02.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.346 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[29191]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:02.346 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:02.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.347 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:02 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:02 smithi045 ceph-mon[33087]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:02.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:02 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:02 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:02 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:02.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:02 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:02.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:02 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:03.456 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:03.164Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000227286s 2024-09-21T09:55:04.403 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:04 smithi032 ceph-mon[40061]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:04.403 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:04 smithi032 ceph-mon[29191]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:04.506 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:04 smithi045 ceph-mon[33087]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[29191]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[40061]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:05.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:05.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:05 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:06.002 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:05 smithi045 ceph-mon[33087]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:06.003 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:05 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:06.003 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:05 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:06.003 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:05 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:06.003 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:05 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:07 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:07 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:07 smithi045 ceph-mon[33087]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:07.811 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:07 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:07.811 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:07 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:07.811 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:07 smithi032 ceph-mon[29191]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:07.811 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:07 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:07.812 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:07 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:07.812 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:07 smithi032 ceph-mon[40061]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:09 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:09.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:55:09] "GET /metrics HTTP/1.1" 200 33802 "" "Prometheus/2.51.0" 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:09.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:09.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:09 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:10.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:10 smithi032 ceph-mon[40061]: Upgrade: Updating grafana.a 2024-09-21T09:55:10.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:10 smithi032 ceph-mon[40061]: Deploying daemon grafana.a on smithi045 2024-09-21T09:55:10.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:10 smithi032 ceph-mon[29191]: Upgrade: Updating grafana.a 2024-09-21T09:55:10.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:10 smithi032 ceph-mon[29191]: Deploying daemon grafana.a on smithi045 2024-09-21T09:55:11.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:10 smithi045 ceph-mon[33087]: Upgrade: Updating grafana.a 2024-09-21T09:55:11.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:10 smithi045 ceph-mon[33087]: Deploying daemon grafana.a on smithi045 2024-09-21T09:55:11.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 09:55:11 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T09:55:11.167Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.0024591s 2024-09-21T09:55:11.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:11 smithi032 ceph-mon[29191]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:11.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:11 smithi032 ceph-mon[29191]: mgrmap e34: y(active, since 92s), standbys: x 2024-09-21T09:55:11.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:11 smithi032 ceph-mon[40061]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:11.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:11 smithi032 ceph-mon[40061]: mgrmap e34: y(active, since 92s), standbys: x 2024-09-21T09:55:12.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:11 smithi045 ceph-mon[33087]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:11 smithi045 ceph-mon[33087]: mgrmap e34: y(active, since 92s), standbys: x 2024-09-21T09:55:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:13 smithi045 ceph-mon[33087]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:13.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:13 smithi032 ceph-mon[29191]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:13 smithi032 ceph-mon[40061]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:16.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:15 smithi045 ceph-mon[33087]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:15 smithi032 ceph-mon[29191]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:15 smithi032 ceph-mon[40061]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:17 smithi045 ceph-mon[33087]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:18.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:17 smithi032 ceph-mon[29191]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:18.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:17 smithi032 ceph-mon[40061]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:19 smithi045 ceph-mon[33087]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:19 smithi032 ceph-mon[29191]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:19 smithi032 ceph-mon[40061]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:19.958 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:55:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:55:19] "GET /metrics HTTP/1.1" 200 33802 "" "Prometheus/2.51.0" 2024-09-21T09:55:20.213 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:19 smithi045 systemd[1]: Stopping Ceph grafana.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:55:20.496 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:20 smithi045 podman[88075]: 2024-09-21 09:55:20.210662741 +0000 UTC m=+0.112487577 container died 419a19b409916fccd763e8b844c03916bb45227a1cc2fd7a0f0555c1cf3b3e19 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container) 2024-09-21T09:55:20.785 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:20 smithi045 podman[88075]: 2024-09-21 09:55:20.633160697 +0000 UTC m=+0.534985534 container cleanup 419a19b409916fccd763e8b844c03916bb45227a1cc2fd7a0f0555c1cf3b3e19 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, architecture=x86_64, description=Ceph Grafana Container, maintainer=Paul Cuzner , release=239.1651231664, vcs-type=git, name=ubi8, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-09-21T09:55:20.785 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:20 smithi045 bash[88075]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a 2024-09-21T09:55:21.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:20 smithi045 podman[88089]: 2024-09-21 09:55:20.826292429 +0000 UTC m=+0.617273505 container remove 419a19b409916fccd763e8b844c03916bb45227a1cc2fd7a0f0555c1cf3b3e19 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, distribution-scope=public, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, maintainer=Paul Cuzner , release=239.1651231664, architecture=x86_64, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, vcs-type=git, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-09-21T09:55:21.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:20 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:55:21.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@grafana.a.service: Failed with result 'exit-code'. 2024-09-21T09:55:21.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 systemd[1]: Stopped Ceph grafana.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:55:21.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@grafana.a.service: Consumed 7.543s CPU time. 2024-09-21T09:55:21.486 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 systemd[1]: Starting Ceph grafana.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:55:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-mon[33087]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:21.785 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 podman[88217]: 2024-09-21 09:55:21.388704727 +0000 UTC m=+0.017544578 image pull 5aad1d7cf004c80a4acb170e1c80fb85781b022f00f08c6bcd22608882b4c748 quay.io/ceph/grafana:10.4.8 2024-09-21T09:55:21.785 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 podman[88217]: 2024-09-21 09:55:21.498763002 +0000 UTC m=+0.127602838 container create 0ad08e43d776ae6e8699fb6d3d73d3840e557d0acc14464f2c8068ad27ad1168 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:55:21.786 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 podman[88217]: 2024-09-21 09:55:21.709242538 +0000 UTC m=+0.338082374 container init 0ad08e43d776ae6e8699fb6d3d73d3840e557d0acc14464f2c8068ad27ad1168 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:55:21.786 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 podman[88217]: 2024-09-21 09:55:21.712278257 +0000 UTC m=+0.341118107 container start 0ad08e43d776ae6e8699fb6d3d73d3840e557d0acc14464f2c8068ad27ad1168 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829616852Z level=info msg="Starting Grafana" version=10.4.8 commit=8c384b71bde142a25ddc01ed5a183ca572b2b7f7 branch=v10.4.x compiled=2024-09-21T09:55:21Z 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829865141Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829875108Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829879948Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.82988419Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829888142Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829891997Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829895978Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.82990033Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829904379Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829908189Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:55:22.073 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829912053Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.82991597Z level=info msg=Target target=[all] 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.82992336Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.82992739Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829931112Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829936137Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829940071Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=settings t=2024-09-21T09:55:21.829943863Z level=info msg="App mode production" 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=sqlstore t=2024-09-21T09:55:21.830277733Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=sqlstore t=2024-09-21T09:55:21.830295126Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.830713216Z level=info msg="Starting DB migrations" 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 bash[88217]: 0ad08e43d776ae6e8699fb6d3d73d3840e557d0acc14464f2c8068ad27ad1168 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 systemd[1]: Started Ceph grafana.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.846259749Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-21T09:55:22.074 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.852436982Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=6.174568ms 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.85293032Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.853081156Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=151.294µs 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.853455601Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.854214397Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=758.354µs 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.854723599Z level=info msg="Executing migration" id="update login field with orgid to allow for multiple service accounts with same name across orgs" 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.854930045Z level=info msg="Migration successfully executed" id="update login field with orgid to allow for multiple service accounts with same name across orgs" duration=206.492µs 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.85550419Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.861158645Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.652152ms 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.861561052Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.867637245Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=6.073716ms 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.86817362Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-21T09:55:22.075 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.874101482Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=5.921882ms 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.874586662Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.874744243Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=159.568µs 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.87511593Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.875919115Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=802.77µs 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.876335265Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.87707843Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=742.511µs 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.877471842Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.883333765Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=5.859158ms 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.883864813Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.889294284Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=5.42648ms 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.889776427Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.889826794Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=50.317µs 2024-09-21T09:55:22.077 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.890229308Z level=info msg="Executing migration" id="add library_element folder uid" 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.896095131Z level=info msg="Migration successfully executed" id="add library_element folder uid" duration=5.866225ms 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.896476451Z level=info msg="Executing migration" id="populate library_element folder_uid" 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.896806065Z level=info msg="Migration successfully executed" id="populate library_element folder_uid" duration=321.952µs 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.897218225Z level=info msg="Executing migration" id="add index library_element org_id-folder_uid-name-kind" 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.898120907Z level=info msg="Migration successfully executed" id="add index library_element org_id-folder_uid-name-kind" duration=902.198µs 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.898543185Z level=info msg="Executing migration" id="permission kind migration" 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.904036657Z level=info msg="Migration successfully executed" id="permission kind migration" duration=5.492543ms 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.9044071Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.909899856Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.493896ms 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.910230086Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.91557348Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.34227ms 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.915927658Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.916757236Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=829.276µs 2024-09-21T09:55:22.078 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.917094097Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.917899639Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=806.448µs 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.918228184Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.918997015Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=768.49µs 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.919332539Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.91935739Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=25.909µs 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.919790413Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.920525538Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=734.353µs 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.920858663Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.921613416Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=754.602µs 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.92193844Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.939554095Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=17.611601ms 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.940236357Z level=info msg="Executing migration" id="create correlation v2" 2024-09-21T09:55:22.079 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.941113099Z level=info msg="Migration successfully executed" id="create correlation v2" duration=876.622µs 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.941459449Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.942152776Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=693.035µs 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.942481162Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.943235804Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=756.888µs 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.9435869Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.944272148Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=684.958µs 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.944646772Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.944818781Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=172.048µs 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.945224913Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.945879192Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=654.746µs 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.946203774Z level=info msg="Executing migration" id="add provisioning column" 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.951755023Z level=info msg="Migration successfully executed" id="add provisioning column" duration=5.551205ms 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.952103526Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-21T09:55:22.080 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.952254506Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=151.011µs 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.952635415Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.95268173Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=46.696µs 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.953101962Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.958753273Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=5.649798ms 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.959100841Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.959311773Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=212.623µs 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.959695918Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.959820094Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=126.333µs 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.960199958Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.961175806Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=976.212µs 2024-09-21T09:55:22.081 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.961513266Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.962244081Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=730.598µs 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.962580344Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.963341918Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=761.792µs 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.963674553Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.964097688Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=421.74µs 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.964426625Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.964633362Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=206.441µs 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.964994108Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.965724103Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=730.986µs 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.966037821Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.966806737Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=768.37µs 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.967115984Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-21T09:55:22.082 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.967830496Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=714.248µs 2024-09-21T09:55:22.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.968140619Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-21T09:55:22.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.968945697Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=806.145µs 2024-09-21T09:55:22.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.969260872Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-21T09:55:22.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.969995059Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=734.064µs 2024-09-21T09:55:22.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.970339997Z level=info msg="Executing migration" id="create anon_device table" 2024-09-21T09:55:22.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.970959818Z level=info msg="Migration successfully executed" id="create anon_device table" duration=618.617µs 2024-09-21T09:55:22.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.971272713Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-21T09:55:22.083 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.971990084Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=717.121µs 2024-09-21T09:55:22.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.972306957Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-21T09:55:22.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.973035618Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=728.446µs 2024-09-21T09:55:22.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.97334423Z level=info msg="Executing migration" id="create signing_key table" 2024-09-21T09:55:22.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.974018294Z level=info msg="Migration successfully executed" id="create signing_key table" duration=673.679µs 2024-09-21T09:55:22.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.974452426Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-21T09:55:22.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.975142507Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=689.731µs 2024-09-21T09:55:22.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.975460038Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-21T09:55:22.084 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.976115692Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=655.538µs 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.976439363Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.97690067Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=461.888µs 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.977262994Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.9831717Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.907445ms 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.983524404Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.985448053Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.92652ms 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.986033321Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.986046841Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=11.628µs 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.986453364Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-21T09:55:22.085 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.987177006Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=724.274µs 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.987497603Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.98750732Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=10.22µs 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.987878569Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.988735325Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=856.227µs 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.989055401Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.989913395Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=857.589µs 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.990244894Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.991100161Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=854.035µs 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.991451139Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.992144516Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=694.781µs 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.992488048Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.992938831Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=451.026µs 2024-09-21T09:55:22.086 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.993258109Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.993299304Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=41.536µs 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.99369381Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.999331215Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.636619ms 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:21.999688466Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:22.005579515Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.890469ms 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:22.00591578Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:22.006146704Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=230.847µs 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=migrator t=2024-09-21T09:55:22.006498135Z level=info msg="migrations completed" performed=69 skipped=482 duration=160.327671ms 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=sqlstore t=2024-09-21T09:55:22.00720552Z level=info msg="Created default organization" 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=secrets t=2024-09-21T09:55:22.009244915Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=plugin.store t=2024-09-21T09:55:22.023876677Z level=info msg="Loading plugins..." 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=local.finder t=2024-09-21T09:55:22.069226892Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-21T09:55:22.087 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=plugin.store t=2024-09-21T09:55:22.069355633Z level=info msg="Plugins loaded" count=55 duration=45.480046ms 2024-09-21T09:55:22.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:21 smithi032 ceph-mon[29191]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:22.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:21 smithi032 ceph-mon[40061]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=query_data t=2024-09-21T09:55:22.074824731Z level=info msg="Query Service initialization" 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=live.push_http t=2024-09-21T09:55:22.078141357Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=ngalert.migration t=2024-09-21T09:55:22.079196915Z level=info msg=Starting 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=ngalert.state.manager t=2024-09-21T09:55:22.0846758Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=infra.usagestats.collector t=2024-09-21T09:55:22.08604718Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=provisioning.datasources t=2024-09-21T09:55:22.088528306Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=provisioning.datasources t=2024-09-21T09:55:22.088827012Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=provisioning.alerting t=2024-09-21T09:55:22.120859281Z level=info msg="starting to provision alerting" 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=provisioning.alerting t=2024-09-21T09:55:22.120876635Z level=info msg="finished to provision alerting" 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=ngalert.state.manager t=2024-09-21T09:55:22.121302308Z level=info msg="Warming state cache for startup" 2024-09-21T09:55:22.354 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=grafanaStorageLogger t=2024-09-21T09:55:22.121517993Z level=info msg="Storage starting" 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=http.server t=2024-09-21T09:55:22.12301422Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=http.server t=2024-09-21T09:55:22.123311885Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:55:22.130719338Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=provisioning.dashboard t=2024-09-21T09:55:22.159671606Z level=info msg="starting to provision dashboards" 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=ngalert.state.manager t=2024-09-21T09:55:22.174330147Z level=info msg="State cache has been initialized" states=0 duration=53.026542ms 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=ngalert.scheduler t=2024-09-21T09:55:22.174360109Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=ticker t=2024-09-21T09:55:22.174415617Z level=info msg=starting first_tick=2024-09-21T09:55:30Z 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=plugins.update.checker t=2024-09-21T09:55:22.197212893Z level=info msg="Update check succeeded" duration=76.034138ms 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=grafana.update.checker t=2024-09-21T09:55:22.208694619Z level=info msg="Update check succeeded" duration=86.552901ms 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=sqlstore.transactions t=2024-09-21T09:55:22.249257002Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:55:22.355 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=provisioning.dashboard t=2024-09-21T09:55:22.352164044Z level=info msg="finished to provision dashboards" 2024-09-21T09:55:22.900 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=grafana-apiserver t=2024-09-21T09:55:22.767079143Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-21T09:55:22.900 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=grafana-apiserver t=2024-09-21T09:55:22.767686292Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-21T09:55:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:22 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:23.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:22 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:23.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:22 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:23.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:22 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:23.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:22 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:23.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:22 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:23.219 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:23.219 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:23.219 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:22 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:23.664 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:55:23.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:23 smithi032 ceph-mon[40061]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:23 smithi032 ceph-mon[29191]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:24.180 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:23 smithi045 ceph-mon[33087]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:24.262 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (23s) 19s ago 6m 15.5M - 0.27.0 11f11916f8cd d6fa4f2dece8 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 starting - - - - 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (4m) 19s ago 4m 16.9M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (4m) 17s ago 4m 17.0M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (86s) 17s ago 9m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (114s) 19s ago 11m 512M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (11m) 19s ago 11m 55.5M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 017caecda0d9 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (10m) 17s ago 10m 47.1M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (10m) 19s ago 10m 43.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 125675da4b84 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (72s) 19s ago 5m 10.0M - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (66s) 17s ago 5m 10.5M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T09:55:24.263 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (9m) 19s ago 9m 58.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:55:24.264 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (9m) 19s ago 9m 52.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:55:24.264 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (8m) 19s ago 8m 57.2M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:55:24.264 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (8m) 19s ago 8m 45.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:55:24.264 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (8m) 17s ago 8m 53.2M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:55:24.264 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (7m) 17s ago 7m 47.8M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:55:24.264 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (7m) 17s ago 7m 46.9M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:55:24.264 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (6m) 17s ago 6m 49.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:55:24.264 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (43s) 17s ago 6m 37.5M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T09:55:24.619 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:55:24.619 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:55:24.619 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:55:24.619 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:55:24.619 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:55:24.619 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:55:24.619 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:55:24.619 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:55:24.620 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:55:24.924 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:55:24.924 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:55:24.924 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:55:24.924 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:55:24.925 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [ 2024-09-21T09:55:24.925 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr" 2024-09-21T09:55:24.925 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T09:55:24.925 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T09:55:24.925 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading grafana daemons", 2024-09-21T09:55:24.925 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:55:24.925 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:55:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:24 smithi045 ceph-mon[33087]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:24 smithi045 ceph-mon[33087]: from='client.25664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:24 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:24 smithi045 ceph-mon[33087]: from='client.25670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:24 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/425297246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[29191]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[29191]: from='client.25664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[29191]: from='client.25670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/425297246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:25.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[40061]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:25.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[40061]: from='client.25664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:25.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:25.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[40061]: from='client.25670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:25.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:24 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/425297246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:25 smithi032 ceph-mon[40061]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:25 smithi032 ceph-mon[40061]: from='client.25682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:25 smithi032 ceph-mon[29191]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:25 smithi032 ceph-mon[29191]: from='client.25682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:25 smithi045 ceph-mon[33087]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:25 smithi045 ceph-mon[33087]: from='client.25682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:27.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:26 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:26 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.216 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.216 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.216 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:27.216 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:26 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:28.201 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:27 smithi045 ceph-mon[33087]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:28.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:27 smithi032 ceph-mon[29191]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:28.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:27 smithi032 ceph-mon[40061]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:29.953 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:55:29] "GET /metrics HTTP/1.1" 200 33798 "" "Prometheus/2.51.0" 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:55:30.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T09:55:30.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:55:30.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:55:30.211 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T09:55:30.211 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:55:30.211 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:55:30.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:30.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T09:55:30.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:55:30.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T09:55:30.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T09:55:30.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T09:55:30.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:55:30.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T09:55:30.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:55:30.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T09:55:30.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:55:30.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:30.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:30.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:30.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:30.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:30.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:30.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:29 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:55:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:55:30.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T09:55:30.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:55:30.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:55:30.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T09:55:30.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:55:30.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T09:55:30.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:30.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:55:30.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:29 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:31.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:30 smithi032 ceph-mon[29191]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:55:31.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:30 smithi032 ceph-mon[29191]: Upgrade: Finalizing container_image settings 2024-09-21T09:55:31.237 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:30 smithi032 ceph-mon[29191]: Upgrade: Complete! 2024-09-21T09:55:31.237 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:30 smithi032 ceph-mon[40061]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:55:31.237 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:30 smithi032 ceph-mon[40061]: Upgrade: Finalizing container_image settings 2024-09-21T09:55:31.237 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:30 smithi032 ceph-mon[40061]: Upgrade: Complete! 2024-09-21T09:55:31.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:30 smithi045 ceph-mon[33087]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:55:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:30 smithi045 ceph-mon[33087]: Upgrade: Finalizing container_image settings 2024-09-21T09:55:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:30 smithi045 ceph-mon[33087]: Upgrade: Complete! 2024-09-21T09:55:32.255 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:31 smithi032 ceph-mon[40061]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:32.255 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:31 smithi032 ceph-mon[29191]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:31 smithi045 ceph-mon[33087]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:33 smithi045 ceph-mon[33087]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:34.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:33 smithi032 ceph-mon[29191]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:34.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:33 smithi032 ceph-mon[40061]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:35 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:35.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:35 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:35.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:35 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:55:36.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:36 smithi032 ceph-mon[29191]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:36 smithi032 ceph-mon[40061]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:36 smithi045 ceph-mon[33087]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:38.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:38 smithi032 ceph-mon[29191]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:38 smithi032 ceph-mon[40061]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:38 smithi045 ceph-mon[33087]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:39.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:39 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:39.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:39 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:39.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:39 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:39.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:55:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:55:39] "GET /metrics HTTP/1.1" 200 33798 "" "Prometheus/2.51.0" 2024-09-21T09:55:40.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:40 smithi032 ceph-mon[29191]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:40.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:40 smithi032 ceph-mon[40061]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:40.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:40 smithi045 ceph-mon[33087]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:42.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:42 smithi032 ceph-mon[40061]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:42.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:42 smithi032 ceph-mon[29191]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:42.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:42 smithi045 ceph-mon[33087]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:44.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:44 smithi032 ceph-mon[40061]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:44 smithi032 ceph-mon[29191]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:44.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:44 smithi045 ceph-mon[33087]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:46.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:46 smithi032 ceph-mon[29191]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:46 smithi032 ceph-mon[40061]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:46 smithi045 ceph-mon[33087]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:48.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:48 smithi032 ceph-mon[29191]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:48.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:48 smithi032 ceph-mon[40061]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:48.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:48 smithi045 ceph-mon[33087]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:55:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:55:49] "GET /metrics HTTP/1.1" 200 33798 "" "Prometheus/2.51.0" 2024-09-21T09:55:50.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:50 smithi032 ceph-mon[29191]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:50.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:50 smithi032 ceph-mon[40061]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:50.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:50 smithi045 ceph-mon[33087]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:52.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:52 smithi032 ceph-mon[40061]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:52 smithi032 ceph-mon[29191]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:52.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:52 smithi045 ceph-mon[33087]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:54 smithi032 ceph-mon[40061]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:54 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:54.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:54 smithi032 ceph-mon[29191]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:54.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:54 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:54 smithi045 ceph-mon[33087]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:54 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:55.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-21T09:55:56.204 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:56.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:56 smithi032 ceph-mon[40061]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:56.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:56 smithi032 ceph-mon[40061]: from='client.15801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:56 smithi032 ceph-mon[29191]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:56 smithi032 ceph-mon[29191]: from='client.15801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:56.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:56 smithi045 ceph-mon[33087]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:56.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:56 smithi045 ceph-mon[33087]: from='client.15801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:57.858 INFO:teuthology.orchestra.run.smithi032.stdout:wait for minority of mons to upgrade 2024-09-21T09:55:58.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:55:58 smithi032 ceph-mon[29191]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:58.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:55:58 smithi032 ceph-mon[40061]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:58.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:55:58 smithi045 ceph-mon[33087]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:58.629 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-21T09:55:58.840 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:59.566 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:55:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:55:59] "GET /metrics HTTP/1.1" 200 33808 "" "Prometheus/2.51.0" 2024-09-21T09:56:00.295 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:00 smithi032 ceph-mon[29191]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:00.296 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:00 smithi032 ceph-mon[40061]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:00.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:00 smithi045 ceph-mon[33087]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:02.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:02 smithi032 ceph-mon[29191]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:02.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:02 smithi032 ceph-mon[29191]: from='client.25694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:02 smithi032 ceph-mon[40061]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:02 smithi032 ceph-mon[40061]: from='client.25694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:02.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:02 smithi045 ceph-mon[33087]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:02.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:02 smithi045 ceph-mon[33087]: from='client.25694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:02.633 INFO:teuthology.orchestra.run.smithi032.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:56:03.286 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:56:03.496 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[40061]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[40061]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:56:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:56:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[40061]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:56:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[29191]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[29191]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:56:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:56:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:03 smithi032 ceph-mon[29191]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:56:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:03 smithi045 ceph-mon[33087]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:03 smithi045 ceph-mon[33087]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:56:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:03 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:03 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:03 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:03 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:56:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:03 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:03 smithi045 ceph-mon[33087]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:56:05.949 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:05 smithi032 ceph-mon[40061]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:05.949 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:05 smithi032 ceph-mon[29191]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:05 smithi045 ceph-mon[33087]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:06.689 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:56:07.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:07.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:56:07.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: Upgrade: Setting container_image for all mgr 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[29191]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:56:07.186 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:07.187 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:56:07.187 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: Upgrade: Setting container_image for all mgr 2024-09-21T09:56:07.187 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:07.187 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:56:07.187 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:56:07.187 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:06 smithi032 ceph-mon[40061]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all mgr 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:56:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:06 smithi045 ceph-mon[33087]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:56:07.299 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (66s) 62s ago 6m 15.5M - 0.27.0 11f11916f8cd d6fa4f2dece8 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (45s) 41s ago 6m 50.2M - 10.4.8 5aad1d7cf004 0ad08e43d776 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (5m) 62s ago 5m 16.9M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (5m) 41s ago 5m 17.1M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (2m) 41s ago 10m 454M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (2m) 62s ago 12m 512M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (12m) 62s ago 12m 55.5M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 017caecda0d9 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (11m) 41s ago 11m 48.0M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (10m) 62s ago 10m 43.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 125675da4b84 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (115s) 62s ago 6m 10.0M - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (109s) 41s ago 6m 10.6M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T09:56:07.300 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (10m) 62s ago 10m 58.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:56:07.301 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (9m) 62s ago 9m 52.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:56:07.301 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (9m) 62s ago 9m 57.2M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:56:07.301 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (9m) 62s ago 9m 45.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:56:07.301 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (8m) 41s ago 8m 53.3M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:56:07.301 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (8m) 41s ago 8m 47.9M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:56:07.301 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (8m) 41s ago 8m 47.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:56:07.301 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (7m) 41s ago 7m 49.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:56:07.301 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (86s) 41s ago 7m 38.8M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:56:07.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:56:07.681 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:56:07.681 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:56:07.681 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:56:07.681 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:56:07.681 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:56:07.999 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:56:07.999 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:56:07.999 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:56:07.999 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-21T09:56:07.999 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T09:56:08.000 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "0/3 daemons upgraded", 2024-09-21T09:56:08.000 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:56:08.000 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:56:08.000 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:56:08.256 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[29191]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:08.256 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[29191]: from='client.15813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:08.256 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[29191]: from='client.25706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:08.256 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[29191]: from='client.25712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:08.256 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:08.256 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[29191]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[29191]: from='client.? 172.21.15.32:0/3361815178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[40061]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[40061]: from='client.15813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[40061]: from='client.25706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[40061]: from='client.25712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:08.257 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:07 smithi032 ceph-mon[40061]: from='client.? 172.21.15.32:0/3361815178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:56:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:07 smithi045 ceph-mon[33087]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:07 smithi045 ceph-mon[33087]: from='client.15813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:07 smithi045 ceph-mon[33087]: from='client.25706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:07 smithi045 ceph-mon[33087]: from='client.25712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:07 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' 2024-09-21T09:56:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:08 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:56:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:08 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:56:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:08 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:08 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3361815178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:56:09.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:09 smithi045 ceph-mon[33087]: Upgrade: Updating mon.a 2024-09-21T09:56:10.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:09 smithi045 ceph-mon[33087]: Deploying daemon mon.a on smithi032 2024-09-21T09:56:10.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:09 smithi045 ceph-mon[33087]: from='client.25724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:10.078 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-mon[40061]: Upgrade: Updating mon.a 2024-09-21T09:56:10.079 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-mon[40061]: Deploying daemon mon.a on smithi032 2024-09-21T09:56:10.079 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-mon[40061]: from='client.25724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:10.079 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-mon[29191]: Upgrade: Updating mon.a 2024-09-21T09:56:10.079 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-mon[29191]: Deploying daemon mon.a on smithi032 2024-09-21T09:56:10.079 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-mon[29191]: from='client.25724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:10.079 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:09 smithi032 systemd[1]: Stopping Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:56:10.080 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:56:09] "GET /metrics HTTP/1.1" 200 33811 "" "Prometheus/2.51.0" 2024-09-21T09:56:10.080 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:09] ENGINE Bus STOPPING 2024-09-21T09:56:10.080 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[29187]: 2024-09-21T09:56:09.629+0000 7fd694b3a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:56:10.080 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[29187]: 2024-09-21T09:56:09.629+0000 7fd694b3a640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T09:56:10.080 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:09 smithi032 podman[101126]: 2024-09-21 09:56:09.651398095 +0000 UTC m=+0.142092287 container died 017caecda0d912368af69361a2cdc938fe1158bfdb788902e6d707c4e3ce0dbb (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:56:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:10 smithi032 podman[101126]: 2024-09-21 09:56:10.130194231 +0000 UTC m=+0.620888466 container cleanup 017caecda0d912368af69361a2cdc938fe1158bfdb788902e6d707c4e3ce0dbb (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=reef-6a1bdf8, ceph=True) 2024-09-21T09:56:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:10 smithi032 bash[101126]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:56:10.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:56:10.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:10] ENGINE Bus STOPPED 2024-09-21T09:56:10.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:10] ENGINE Bus STARTING 2024-09-21T09:56:10.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:10] ENGINE Serving on http://:::9283 2024-09-21T09:56:10.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:10] ENGINE Bus STARTED 2024-09-21T09:56:10.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:10 smithi032 podman[101141]: 2024-09-21 09:56:10.423304936 +0000 UTC m=+0.771386855 container remove 017caecda0d912368af69361a2cdc938fe1158bfdb788902e6d707c4e3ce0dbb (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:56:10.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:10 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:56:10.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:10 smithi032 systemd[1]: Stopped Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:56:10.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:10 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service: Consumed 14.559s CPU time. 2024-09-21T09:56:11.123 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:10 smithi032 systemd[1]: Starting Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:56:11.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:10 smithi032 podman[101280]: 2024-09-21 09:56:10.898549361 +0000 UTC m=+0.016596074 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:56:11.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:10 smithi032 podman[101280]: 2024-09-21 09:56:10.999731604 +0000 UTC m=+0.117778304 container create 525e81f3c17e3af3fbb410b4ee38f2dd0968b23a35661aea237358d815faa740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T09:56:11.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 podman[101280]: 2024-09-21 09:56:11.251981007 +0000 UTC m=+0.370027710 container init 525e81f3c17e3af3fbb410b4ee38f2dd0968b23a35661aea237358d815faa740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:56:11.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 podman[101280]: 2024-09-21 09:56:11.255510115 +0000 UTC m=+0.373556813 container start 525e81f3c17e3af3fbb410b4ee38f2dd0968b23a35661aea237358d815faa740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 2 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: pidfile_write: ignore empty --pid-file 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: load: jerasure load: lrc 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Git sha 0 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: DB SUMMARY 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: DB Session ID: IGQADX5FAEK9TEFFWBMX 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1511 Bytes 2024-09-21T09:56:11.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000032.sst 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000030.log size: 1807496 ; 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.env: 0x5589e6f6d120 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.info_log: 0x5589e7f19700 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.statistics: (nil) 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.use_fsync: 0 2024-09-21T09:56:11.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.db_log_dir: 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.wal_dir: 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:56:11.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.write_buffer_manager: 0x5589e7f1db80 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:56:11.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:56:11.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.unordered_write: 0 2024-09-21T09:56:11.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:56:11.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:56:11.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:56:11.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:56:11.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.row_cache: None 2024-09-21T09:56:11.462 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.wal_filter: None 2024-09-21T09:56:11.463 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:56:11.463 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:56:11.463 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.wal_compression: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:56:11.464 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:56:11.465 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_open_files: -1 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Compression algorithms supported: 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: kZSTD supported: 0 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:56:11.466 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.merge_operator: 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_filter: None 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5589e7f19320) 2024-09-21T09:56:11.467 INFO:journalctl@ceph.mon.a.smithi032.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:56:11.468 INFO:journalctl@ceph.mon.a.smithi032.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:56:11.468 INFO:journalctl@ceph.mon.a.smithi032.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:56:11.468 INFO:journalctl@ceph.mon.a.smithi032.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:56:11.468 INFO:journalctl@ceph.mon.a.smithi032.stdout: index_type: 0 2024-09-21T09:56:11.468 INFO:journalctl@ceph.mon.a.smithi032.stdout: data_block_index_type: 0 2024-09-21T09:56:11.468 INFO:journalctl@ceph.mon.a.smithi032.stdout: index_shortening: 1 2024-09-21T09:56:11.468 INFO:journalctl@ceph.mon.a.smithi032.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:56:11.468 INFO:journalctl@ceph.mon.a.smithi032.stdout: checksum: 4 2024-09-21T09:56:11.468 INFO:journalctl@ceph.mon.a.smithi032.stdout: no_block_cache: 0 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache: 0x5589e7f0d610 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache_options: 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: capacity : 536870912 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: num_shard_bits : 4 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: strict_capacity_limit : 0 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_cache_compressed: (nil) 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: persistent_cache: (nil) 2024-09-21T09:56:11.469 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_size: 4096 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_size_deviation: 10 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_restart_interval: 16 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: index_block_restart_interval: 1 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: metadata_block_size: 4096 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: partition_filters: 0 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: use_delta_encoding: 1 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: filter_policy: bloomfilter 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: whole_key_filtering: 1 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: verify_compression: 0 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: format_version: 5 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: enable_index_compression: 1 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: block_align: 0 2024-09-21T09:56:11.470 INFO:journalctl@ceph.mon.a.smithi032.stdout: max_auto_readahead_size: 262144 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout: prepopulate_block_cache: 0 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression: NoCompression 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.num_levels: 7 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:56:11.471 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:56:11.472 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:56:11.473 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:56:11.474 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:56:11.475 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.ttl: 2592000 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:56:11.476 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 34, last_sequence is 12305, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 30 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3a81636a-4603-4766-8cf1-db3baa1b39f3 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912571283817, "job": 1, "event": "recovery_started", "wal_files": [30]} 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912571292686, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 1617059, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12306, "largest_seqno": 13649, "table_properties": {"data_size": 1610619, "index_size": 3863, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1541, "raw_key_size": 14486, "raw_average_key_size": 23, "raw_value_size": 1597502, "raw_average_value_size": 2636, "num_data_blocks": 177, "num_entries": 606, "num_filter_entries": 606, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726912571, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3a81636a-4603-4766-8cf1-db3baa1b39f3", "db_session_id": "IGQADX5FAEK9TEFFWBMX", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912571292816, "job": 1, "event": "recovery_finished"} 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/version_set.cc:5047] Creating manifest 37 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5589e7f3ce00 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: DB pointer 0x5589e803e000 2024-09-21T09:56:11.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** DB Stats ** 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** Compaction Stats [default] ** 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: L0 1/0 1.54 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 401.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T09:56:11.478 INFO:journalctl@ceph.mon.a.smithi032.stdout: L6 1/0 7.80 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: Sum 2/0 9.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 401.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 401.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** Compaction Stats [default] ** 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 401.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:56:11.479 INFO:journalctl@ceph.mon.a.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:56:11.480 INFO:journalctl@ceph.mon.a.smithi032.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-21T09:56:11.480 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:56:11.480 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:56:11.480 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:56:11.480 INFO:journalctl@ceph.mon.a.smithi032.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:56:11.481 INFO:journalctl@ceph.mon.a.smithi032.stdout: Cumulative compaction: 0.00 GB write, 131.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:56:11.481 INFO:journalctl@ceph.mon.a.smithi032.stdout: Interval compaction: 0.00 GB write, 131.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:56:11.481 INFO:journalctl@ceph.mon.a.smithi032.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:56:11.481 INFO:journalctl@ceph.mon.a.smithi032.stdout: Block cache BinnedLRUCache@0x5589e7f0d610#2 capacity: 512.00 MB usage: 5.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: starting mon.a rank 0 at public addrs [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] at bind addrs [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: mon.a@-1(???) e3 preinit fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: mon.a@-1(???).mds e9 new map 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: mon.a@-1(???).mds e9 print_map 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: e9 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: legacy client fscid: 1 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: Filesystem 'foo' (1) 2024-09-21T09:56:11.482 INFO:journalctl@ceph.mon.a.smithi032.stdout: fs_name foo 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: epoch 8 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: created 2024-09-21T09:50:22.390699+0000 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: modified 2024-09-21T09:50:36.781550+0000 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: tableserver 0 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: root 0 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: session_timeout 60 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: session_autoclose 300 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: max_file_size 1099511627776 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: max_xattr_size 65536 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: required_client_features {} 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: last_failure 0 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: last_failure_osd_epoch 0 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:56:11.483 INFO:journalctl@ceph.mon.a.smithi032.stdout: max_mds 1 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: in 0 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: up {0=14655} 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: failed 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: damaged 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: stopped 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: data_pools [4] 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: metadata_pool 3 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: inline_data disabled 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: balancer 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: bal_rank_mask -1 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: standby_count_wanted 1 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: [mds.foo.smithi032.fdolcm{0:14655} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:56:11.484 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout: Standby daemons: 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout: 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout: [mds.foo.smithi045.jnfsfb{-1:24496} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.45:6832/3743591142,v1:172.21.15.45:6833/3743591142] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: mon.a@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 ceph-mon[101299]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 bash[101280]: 525e81f3c17e3af3fbb410b4ee38f2dd0968b23a35661aea237358d815faa740 2024-09-21T09:56:11.485 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:11 smithi032 systemd[1]: Started Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:56:13.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: ignoring --setuser ceph since I am not root 2024-09-21T09:56:13.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: ignoring --setgroup ceph since I am not root 2024-09-21T09:56:13.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mgr[76153]: -- 172.21.15.45:0/520018737 <== mon.1 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558ffa1490e0 con 0x558ffa126800 2024-09-21T09:56:13.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: mon.a calling monitor election 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: monmap epoch 3 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: last_changed 2024-09-21T09:45:13.133999+0000 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: created 2024-09-21T09:43:34.453975+0000 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: min_mon_release 18 (reef) 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: election_strategy: 1 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-09-21T09:56:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: 1: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.b 2024-09-21T09:56:13.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:56:13.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:56:13.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:56:13.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: mgrmap e34: y(active, since 2m), standbys: x 2024-09-21T09:56:13.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: overall HEALTH_OK 2024-09-21T09:56:13.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: from='mgr.25520 ' entity='' 2024-09-21T09:56:13.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:12 smithi045 ceph-mon[33087]: mgrmap e35: y(active, since 2m), standbys: x 2024-09-21T09:56:13.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ignoring --setuser ceph since I am not root 2024-09-21T09:56:13.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ignoring --setgroup ceph since I am not root 2024-09-21T09:56:13.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mgr[85239]: -- 172.21.15.32:0/2454194805 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eddb0710e0 con 0x55eddb04e800 2024-09-21T09:56:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:56:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: mon.a calling monitor election 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: monmap epoch 3 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: last_changed 2024-09-21T09:45:13.133999+0000 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: created 2024-09-21T09:43:34.453975+0000 2024-09-21T09:56:13.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: min_mon_release 18 (reef) 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: election_strategy: 1 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: 1: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.b 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: mgrmap e34: y(active, since 2m), standbys: x 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: overall HEALTH_OK 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: from='mgr.25520 ' entity='' 2024-09-21T09:56:13.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[40061]: mgrmap e35: y(active, since 2m), standbys: x 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: from='mgr.25520 172.21.15.32:0/2656344159' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: mon.a calling monitor election 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: monmap epoch 3 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:13.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: last_changed 2024-09-21T09:45:13.133999+0000 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: created 2024-09-21T09:43:34.453975+0000 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: min_mon_release 18 (reef) 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: election_strategy: 1 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: 1: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.b 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: mgrmap e34: y(active, since 2m), standbys: x 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: overall HEALTH_OK 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: from='mgr.25520 ' entity='' 2024-09-21T09:56:13.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:12 smithi032 ceph-mon[101299]: mgrmap e35: y(active, since 2m), standbys: x 2024-09-21T09:56:18.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: [21/Sep/2024:09:56:18] ENGINE Bus STARTING 2024-09-21T09:56:18.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: CherryPy Checker: 2024-09-21T09:56:18.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: The Application mounted at '' has an empty config. 2024-09-21T09:56:18.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: 2024-09-21T09:56:18.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: [21/Sep/2024:09:56:18] ENGINE Serving on http://:::9283 2024-09-21T09:56:18.285 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x[76149]: [21/Sep/2024:09:56:18] ENGINE Bus STARTED 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: Active manager daemon y restarted 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: Activating manager daemon y 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: mgrmap e36: y(active, starting, since 0.00676895s), standbys: x 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:56:18.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: Manager daemon y is now available 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: Standby manager daemon x restarted 2024-09-21T09:56:18.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: Standby manager daemon x started 2024-09-21T09:56:18.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:56:18.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:56:18.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:56:18.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:18 smithi045 ceph-mon[33087]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:56:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: Active manager daemon y restarted 2024-09-21T09:56:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: Activating manager daemon y 2024-09-21T09:56:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:56:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: mgrmap e36: y(active, starting, since 0.00676895s), standbys: x 2024-09-21T09:56:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:56:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:56:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:56:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:56:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:56:18.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: Manager daemon y is now available 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: Standby manager daemon x restarted 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: Standby manager daemon x started 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[40061]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: Active manager daemon y restarted 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: Activating manager daemon y 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:56:18.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: mgrmap e36: y(active, starting, since 0.00676895s), standbys: x 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi032.fdolcm"}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi045.jnfsfb"}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:56:18.460 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: Manager daemon y is now available 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: Standby manager daemon x restarted 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: Standby manager daemon x started 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:56:18.461 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-mon[101299]: from='mgr.? 172.21.15.45:0/3307796211' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:56:18.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.492+0000 7fce436fc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:18.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.492+0000 7fce436fc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:18.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.492+0000 7fce436fc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:18.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.492+0000 7fce436fc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:18.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.492+0000 7fce436fc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:18.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.494+0000 7fce46f03640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:18.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.494+0000 7fce46f03640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:18.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.494+0000 7fce46f03640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:18.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.494+0000 7fce46f03640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:18.809 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.494+0000 7fce46f03640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:56:19.116 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:19.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:56:19.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:56:19.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:56:19.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:56:19.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:56:19.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:56:19.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[40061]: mgrmap e37: y(active, since 1.01069s), standbys: x 2024-09-21T09:56:19.117 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:18 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:18.992+0000 7fce75a2e640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-21T09:56:19.117 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:19] ENGINE Bus STARTING 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: CherryPy Checker: 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: The Application mounted at '' has an empty config. 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:19] ENGINE Serving on http://:::9283 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:19] ENGINE Bus STARTED 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:56:19.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:56:19.119 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:56:19.119 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-mon[101299]: mgrmap e37: y(active, since 1.01069s), standbys: x 2024-09-21T09:56:19.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:19 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:19.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:19 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:56:19.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:19 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:56:19.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:19 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:56:19.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:19 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:56:19.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:19 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:56:19.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:19 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:56:19.413 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:19 smithi045 ceph-mon[33087]: mgrmap e37: y(active, since 1.01069s), standbys: x 2024-09-21T09:56:19.819 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:56:19] "GET /metrics HTTP/1.1" 200 31113 "" "Prometheus/2.51.0" 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:20 smithi032 ceph-mon[101299]: [21/Sep/2024:09:56:19] ENGINE Bus STARTING 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:20 smithi032 ceph-mon[101299]: [21/Sep/2024:09:56:19] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:20 smithi032 ceph-mon[101299]: [21/Sep/2024:09:56:19] ENGINE Error in HTTPServer.serve 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:56:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: self.do_handshake() 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.a.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:20 smithi032 ceph-mon[40061]: [21/Sep/2024:09:56:19] ENGINE Bus STARTING 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:20 smithi032 ceph-mon[40061]: [21/Sep/2024:09:56:19] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:56:20.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:20 smithi032 ceph-mon[40061]: [21/Sep/2024:09:56:19] ENGINE Error in HTTPServer.serve 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: Traceback (most recent call last): 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._run(expiration_interval) 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: s = self.context.wrap_socket( 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: return self.sslsocket_class._create( 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:56:20.209 INFO:journalctl@ceph.mon.c.smithi032.stdout: self.do_handshake() 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mon.c.smithi032.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mon.c.smithi032.stdout: self._sslobj.do_handshake() 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mon.c.smithi032.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:19.794+0000 7fce2950d640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:56:19] ENGINE Error in HTTPServer.serve 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: Traceback (most recent call last): 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: self._connections.run(self.expiration_interval) 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: self._run(expiration_interval) 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:56:20.210 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: s = self.context.wrap_socket( 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: return self.sslsocket_class._create( 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: self.do_handshake() 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: self._sslobj.do_handshake() 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:56:20.211 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:20 smithi045 ceph-mon[33087]: [21/Sep/2024:09:56:19] ENGINE Bus STARTING 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:20 smithi045 ceph-mon[33087]: [21/Sep/2024:09:56:19] ENGINE Serving on https://172.21.15.32:7150 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:20 smithi045 ceph-mon[33087]: [21/Sep/2024:09:56:19] ENGINE Error in HTTPServer.serve 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout: Traceback (most recent call last): 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._run(expiration_interval) 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:56:20.340 INFO:journalctl@ceph.mon.b.smithi045.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: s = self.context.wrap_socket( 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: return self.sslsocket_class._create( 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: self.do_handshake() 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: self._sslobj.do_handshake() 2024-09-21T09:56:20.341 INFO:journalctl@ceph.mon.b.smithi045.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:56:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:21 smithi045 ceph-mon[33087]: [21/Sep/2024:09:56:19] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:56:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:21 smithi045 ceph-mon[33087]: [21/Sep/2024:09:56:19] ENGINE Bus STARTED 2024-09-21T09:56:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:21 smithi045 ceph-mon[33087]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:21 smithi045 ceph-mon[33087]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:56:21.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:21 smithi032 ceph-mon[101299]: [21/Sep/2024:09:56:19] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:56:21.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:21 smithi032 ceph-mon[101299]: [21/Sep/2024:09:56:19] ENGINE Bus STARTED 2024-09-21T09:56:21.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:21 smithi032 ceph-mon[101299]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:21.430 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:21 smithi032 ceph-mon[101299]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:56:21.431 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:21 smithi032 ceph-mon[40061]: [21/Sep/2024:09:56:19] ENGINE Serving on http://172.21.15.32:8765 2024-09-21T09:56:21.431 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:21 smithi032 ceph-mon[40061]: [21/Sep/2024:09:56:19] ENGINE Bus STARTED 2024-09-21T09:56:21.431 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:21 smithi032 ceph-mon[40061]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:21.431 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:21 smithi032 ceph-mon[40061]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:56:23.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[101299]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[101299]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[40061]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[40061]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:23 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:23 smithi045 ceph-mon[33087]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:23 smithi045 ceph-mon[33087]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:56:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:23 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:23 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:23 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:23 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:24 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:24 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:24 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:24 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:24 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:24 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:24 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:24 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.252 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:24 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.252 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:24 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.252 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:24 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:24.252 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:24 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:25.618 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:25 smithi045 ceph-mon[33087]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:25.660 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:25 smithi032 ceph-mon[40061]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:25.660 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:25 smithi032 ceph-mon[101299]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:27.705 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:56:27.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:56:27.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:56:27.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:27 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: Updating smithi032:/etc/ceph/ceph.conf 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:56:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:56:27.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:27 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-21T09:56:28.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:28 smithi032 ceph-mon[40061]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:56:28.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:28 smithi032 ceph-mon[40061]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:56:28.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:28 smithi032 ceph-mon[40061]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:56:28.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:28 smithi032 ceph-mon[101299]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:56:28.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:28 smithi032 ceph-mon[101299]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:56:28.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:28 smithi032 ceph-mon[101299]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:56:28.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:28 smithi045 ceph-mon[33087]: Updating smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:56:28.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:28 smithi045 ceph-mon[33087]: Updating smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:56:28.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:28 smithi045 ceph-mon[33087]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:56:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:29 smithi045 ceph-mon[33087]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-21T09:56:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:29 smithi045 ceph-mon[33087]: Upgrade: Updating mon.c 2024-09-21T09:56:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:29 smithi045 ceph-mon[33087]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:29 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:56:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:29 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:56:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:29 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:29 smithi045 ceph-mon[33087]: Deploying daemon mon.c on smithi032 2024-09-21T09:56:29.808 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:56:29] "GET /metrics HTTP/1.1" 200 31113 "" "Prometheus/2.51.0" 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[40061]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[40061]: Upgrade: Updating mon.c 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[40061]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[40061]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[40061]: Deploying daemon mon.c on smithi032 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[101299]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[101299]: Upgrade: Updating mon.c 2024-09-21T09:56:29.809 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[101299]: from='mgr.25736 ' entity='mgr.y' 2024-09-21T09:56:29.810 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:56:29.810 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:56:29.810 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:56:29.810 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:29 smithi032 ceph-mon[101299]: Deploying daemon mon.c on smithi032 2024-09-21T09:56:30.372 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:30 smithi032 systemd[1]: Stopping Ceph mon.c for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:56:30.648 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:30 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c[40057]: 2024-09-21T09:56:30.370+0000 7faaf0b25640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:56:30.648 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:30 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c[40057]: 2024-09-21T09:56:30.370+0000 7faaf0b25640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:56:30.649 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:30 smithi032 podman[105570]: 2024-09-21 09:56:30.5538136 +0000 UTC m=+0.304069254 container died 125675da4b8420a5ce1f33b4822cb4ddcd4062f3126d03e4427027f306416813 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T09:56:30.649 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:30 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:30] ENGINE Bus STOPPING 2024-09-21T09:56:30.649 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:30 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:56:30.649 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:30 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:30] ENGINE Bus STOPPED 2024-09-21T09:56:30.649 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:30 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:30] ENGINE Bus STARTING 2024-09-21T09:56:30.649 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:30 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:30] ENGINE Serving on http://:::9283 2024-09-21T09:56:30.649 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:30 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:09:56:30] ENGINE Bus STARTED 2024-09-21T09:56:31.413 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:31 smithi032 podman[105570]: 2024-09-21 09:56:31.045474892 +0000 UTC m=+0.795730582 container cleanup 125675da4b8420a5ce1f33b4822cb4ddcd4062f3126d03e4427027f306416813 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T09:56:31.413 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:31 smithi032 bash[105570]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c 2024-09-21T09:56:31.413 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:31 smithi032 podman[105583]: 2024-09-21 09:56:31.280046828 +0000 UTC m=+0.728121666 container remove 125675da4b8420a5ce1f33b4822cb4ddcd4062f3126d03e4427027f306416813 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T09:56:31.701 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:31 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.c.service: Deactivated successfully. 2024-09-21T09:56:31.974 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:31 smithi032 systemd[1]: Stopped Ceph mon.c for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:56:31.975 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:31 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.c.service: Consumed 7.843s CPU time. 2024-09-21T09:56:31.975 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:31 smithi032 systemd[1]: Starting Ceph mon.c for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:56:31.975 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:31 smithi032 podman[105721]: 2024-09-21 09:56:31.8482305 +0000 UTC m=+0.103482387 container create 233d45e2730d79fdf72f36f6405f3d5b5b06b13a52c98cb5cd6c2cee0fb44a94 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T09:56:31.975 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:31 smithi032 podman[105721]: 2024-09-21 09:56:31.761784452 +0000 UTC m=+0.017036355 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:56:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 podman[105721]: 2024-09-21 09:56:32.092521506 +0000 UTC m=+0.347773394 container init 233d45e2730d79fdf72f36f6405f3d5b5b06b13a52c98cb5cd6c2cee0fb44a94 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-21T09:56:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 podman[105721]: 2024-09-21 09:56:32.09657748 +0000 UTC m=+0.351829378 container start 233d45e2730d79fdf72f36f6405f3d5b5b06b13a52c98cb5cd6c2cee0fb44a94 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T09:56:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:56:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 2 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: pidfile_write: ignore empty --pid-file 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: load: jerasure load: lrc 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Git sha 0 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: DB SUMMARY 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: DB Session ID: FMADZ2HZ51W05JLWIM2T 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1404 Bytes 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 5536053 ; 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:56:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:56:32.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:56:32.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:56:32.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:56:32.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.env: 0x55db58e86120 2024-09-21T09:56:32.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:56:32.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.info_log: 0x55db5aabd700 2024-09-21T09:56:32.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.statistics: (nil) 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.use_fsync: 0 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:56:32.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.db_log_dir: 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.wal_dir: 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:56:32.461 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.write_buffer_manager: 0x55db5aac1b80 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.unordered_write: 0 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:56:32.462 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:56:32.463 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.row_cache: None 2024-09-21T09:56:32.463 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.wal_filter: None 2024-09-21T09:56:32.463 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:56:32.463 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:56:32.463 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:56:32.463 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:56:32.463 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.wal_compression: 0 2024-09-21T09:56:32.463 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:56:32.464 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_open_files: -1 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:56:32.465 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Compression algorithms supported: 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: kZSTD supported: 0 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:56:32.466 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.merge_operator: 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_filter: None 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55db5aabd320) 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:56:32.467 INFO:journalctl@ceph.mon.c.smithi032.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: index_type: 0 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: data_block_index_type: 0 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: index_shortening: 1 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: checksum: 4 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: no_block_cache: 0 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_cache: 0x55db5aab1610 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_cache_options: 2024-09-21T09:56:32.468 INFO:journalctl@ceph.mon.c.smithi032.stdout: capacity : 536870912 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: num_shard_bits : 4 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: strict_capacity_limit : 0 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_cache_compressed: (nil) 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: persistent_cache: (nil) 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_size: 4096 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_size_deviation: 10 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_restart_interval: 16 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: index_block_restart_interval: 1 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: metadata_block_size: 4096 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: partition_filters: 0 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: use_delta_encoding: 1 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: filter_policy: bloomfilter 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: whole_key_filtering: 1 2024-09-21T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi032.stdout: verify_compression: 0 2024-09-21T09:56:32.470 INFO:journalctl@ceph.mon.c.smithi032.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:56:32.470 INFO:journalctl@ceph.mon.c.smithi032.stdout: format_version: 5 2024-09-21T09:56:32.470 INFO:journalctl@ceph.mon.c.smithi032.stdout: enable_index_compression: 1 2024-09-21T09:56:32.470 INFO:journalctl@ceph.mon.c.smithi032.stdout: block_align: 0 2024-09-21T09:56:32.470 INFO:journalctl@ceph.mon.c.smithi032.stdout: max_auto_readahead_size: 262144 2024-09-21T09:56:32.470 INFO:journalctl@ceph.mon.c.smithi032.stdout: prepopulate_block_cache: 0 2024-09-21T09:56:32.471 INFO:journalctl@ceph.mon.c.smithi032.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:56:32.471 INFO:journalctl@ceph.mon.c.smithi032.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:56:32.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:56:32.471 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression: NoCompression 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.num_levels: 7 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:56:32.472 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:56:32.473 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:56:32.474 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:56:32.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:56:32.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:56:32.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:56:32.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:56:32.475 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:56:32.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:56:32.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:56:32.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:56:32.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:56:32.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:56:32.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:56:32.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:56:32.476 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:56:32.477 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.ttl: 2592000 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:56:32.478 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 12918, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 10357bed-1921-4fe3-96ca-02d127d4b05a 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912592124573, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912592137670, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3392318, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12923, "largest_seqno": 14878, "table_properties": {"data_size": 3383854, "index_size": 5370, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2053, "raw_key_size": 19511, "raw_average_key_size": 23, "raw_value_size": 3366215, "raw_average_value_size": 4110, "num_data_blocks": 245, "num_entries": 819, "num_filter_entries": 819, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726912592, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "10357bed-1921-4fe3-96ca-02d127d4b05a", "db_session_id": "FMADZ2HZ51W05JLWIM2T", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912592137818, "job": 1, "event": "recovery_finished"} 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55db5aae0e00 2024-09-21T09:56:32.479 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: DB pointer 0x55db5abe2000 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: ** DB Stats ** 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: ** Compaction Stats [default] ** 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:56:32.480 INFO:journalctl@ceph.mon.c.smithi032.stdout: L0 1/0 3.24 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 622.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: L6 1/0 7.80 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: Sum 2/0 11.04 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 622.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 622.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: ** Compaction Stats [default] ** 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 622.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:56:32.481 INFO:journalctl@ceph.mon.c.smithi032.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: Cumulative compaction: 0.00 GB write, 202.88 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: Interval compaction: 0.00 GB write, 202.88 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: Block cache BinnedLRUCache@0x55db5aab1610#2 capacity: 512.00 MB usage: 8.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.28 KB,0.000435114%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: starting mon.c rank 2 at public addrs [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] at bind addrs [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: mon.c@-1(???) e3 preinit fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: mon.c@-1(???).mds e9 new map 2024-09-21T09:56:32.482 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: mon.c@-1(???).mds e9 print_map 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: e9 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: legacy client fscid: 1 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: Filesystem 'foo' (1) 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: fs_name foo 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: epoch 8 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: created 2024-09-21T09:50:22.390699+0000 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: modified 2024-09-21T09:50:36.781550+0000 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: tableserver 0 2024-09-21T09:56:32.483 INFO:journalctl@ceph.mon.c.smithi032.stdout: root 0 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: session_timeout 60 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: session_autoclose 300 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: max_file_size 1099511627776 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: max_xattr_size 65536 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: required_client_features {} 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: last_failure 0 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: last_failure_osd_epoch 0 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: max_mds 1 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: in 0 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: up {0=14655} 2024-09-21T09:56:32.484 INFO:journalctl@ceph.mon.c.smithi032.stdout: failed 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: damaged 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: stopped 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: data_pools [4] 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: metadata_pool 3 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: inline_data disabled 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: balancer 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: bal_rank_mask -1 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: standby_count_wanted 1 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: [mds.foo.smithi032.fdolcm{0:14655} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: Standby daemons: 2024-09-21T09:56:32.485 INFO:journalctl@ceph.mon.c.smithi032.stdout: 2024-09-21T09:56:32.486 INFO:journalctl@ceph.mon.c.smithi032.stdout: [mds.foo.smithi045.jnfsfb{-1:24496} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.45:6832/3743591142,v1:172.21.15.45:6833/3743591142] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:56:32.486 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: mon.c@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T09:56:32.486 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:56:32.486 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:56:32.486 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:56:32.486 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 ceph-mon[105748]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-21T09:56:32.486 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 bash[105721]: 233d45e2730d79fdf72f36f6405f3d5b5b06b13a52c98cb5cd6c2cee0fb44a94 2024-09-21T09:56:32.486 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:32 smithi032 systemd[1]: Started Ceph mon.c for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:56:38.321 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: mon.c calling monitor election 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: mon.a calling monitor election 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: mon.b calling monitor election 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: monmap epoch 3 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: last_changed 2024-09-21T09:45:13.133999+0000 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: created 2024-09-21T09:43:34.453975+0000 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: min_mon_release 18 (reef) 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: election_strategy: 1 2024-09-21T09:56:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: 1: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.b 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: mgrmap e39: y(active, since 20s), standbys: x 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[101299]: overall HEALTH_OK 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: mon.c calling monitor election 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: mon.a calling monitor election 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: mon.b calling monitor election 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: monmap epoch 3 2024-09-21T09:56:38.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: last_changed 2024-09-21T09:45:13.133999+0000 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: created 2024-09-21T09:43:34.453975+0000 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: min_mon_release 18 (reef) 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: election_strategy: 1 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: 1: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.b 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: mgrmap e39: y(active, since 20s), standbys: x 2024-09-21T09:56:38.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:38 smithi032 ceph-mon[105748]: overall HEALTH_OK 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: mon.c calling monitor election 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: mon.a calling monitor election 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: mon.b calling monitor election 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: monmap epoch 3 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: last_changed 2024-09-21T09:45:13.133999+0000 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: created 2024-09-21T09:43:34.453975+0000 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: min_mon_release 18 (reef) 2024-09-21T09:56:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: election_strategy: 1 2024-09-21T09:56:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-09-21T09:56:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: 1: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.b 2024-09-21T09:56:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:56:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:56:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:56:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: mgrmap e39: y(active, since 20s), standbys: x 2024-09-21T09:56:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:38 smithi045 ceph-mon[33087]: overall HEALTH_OK 2024-09-21T09:56:38.944 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:56:38.944 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (98s) 16s ago 7m 20.0M - 0.27.0 11f11916f8cd d6fa4f2dece8 2024-09-21T09:56:38.944 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (77s) 16s ago 7m 71.4M - 10.4.8 5aad1d7cf004 0ad08e43d776 2024-09-21T09:56:38.944 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (6m) 16s ago 6m 17.7M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:56:38.944 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (6m) 16s ago 6m 17.5M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:56:38.944 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (2m) 16s ago 11m 454M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T09:56:38.944 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (3m) 16s ago 12m 488M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (28s) 16s ago 13m 29.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 525e81f3c17e 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (11m) 16s ago 11m 54.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 starting - - - 2048M 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (2m) 16s ago 6m 10.9M - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (2m) 16s ago 6m 10.3M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (10m) 16s ago 10m 58.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (10m) 16s ago 10m 53.1M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (10m) 16s ago 10m 58.2M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (9m) 16s ago 9m 45.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (9m) 16s ago 9m 54.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (8m) 16s ago 8m 48.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (8m) 16s ago 8m 47.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (8m) 16s ago 8m 50.2M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:56:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (117s) 16s ago 7m 41.0M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T09:56:39.352 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:56:39.352 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:56:39.352 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 1, 2024-09-21T09:56:39.352 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:56:39.352 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:56:39.352 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 11, 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:56:39.353 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:56:39.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:39 smithi032 ceph-mon[105748]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:39.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:39 smithi032 ceph-mon[105748]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:39.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:39 smithi032 ceph-mon[105748]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:39.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4109397684' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:39.675 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:56:39] "GET /metrics HTTP/1.1" 200 33802 "" "Prometheus/2.51.0" 2024-09-21T09:56:39.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:39 smithi032 ceph-mon[101299]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:39.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:39 smithi032 ceph-mon[101299]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:39.676 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:39 smithi032 ceph-mon[101299]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:39.676 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4109397684' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:39.679 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:56:39.679 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:56:39.679 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:56:39.679 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-09-21T09:56:39.679 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T09:56:39.680 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "1/3 daemons upgraded", 2024-09-21T09:56:39.680 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:56:39.680 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:56:39.680 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:56:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:39 smithi045 ceph-mon[33087]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:39 smithi045 ceph-mon[33087]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:39 smithi045 ceph-mon[33087]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:39 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/4109397684' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:40.672 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:40 smithi032 ceph-mon[105748]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:40.673 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:40 smithi032 ceph-mon[105748]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:40.673 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:40 smithi032 ceph-mon[101299]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:40.673 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:40 smithi032 ceph-mon[101299]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:40.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:40 smithi045 ceph-mon[33087]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:40 smithi045 ceph-mon[33087]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:41.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:41 smithi032 ceph-mon[101299]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:41 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:41 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:41 smithi032 ceph-mon[105748]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:41 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:41 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:41 smithi045 ceph-mon[33087]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:41 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:41 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:42.256 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:42 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:56:42.147+0000 7fce75a2e640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T09:56:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:42 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:42 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:43.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:43 smithi032 ceph-mon[101299]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:43.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:43 smithi032 ceph-mon[105748]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:44.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:43 smithi045 ceph-mon[33087]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:56:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:56:45.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:56:45.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:56:45.965 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:45.966 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:45.967 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:45.967 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:45.967 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:46.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:46.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:46.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:46.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:46.040 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:46.040 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:46.040 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:46.040 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:56:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:56:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:56:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:56:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all grafana 2024-09-21T09:56:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all loki 2024-09-21T09:56:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all promtail 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[101299]: Upgrade: Finalizing container_image settings 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[101299]: Upgrade: Complete! 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all grafana 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all loki 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all promtail 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[105748]: Upgrade: Finalizing container_image settings 2024-09-21T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:46 smithi032 ceph-mon[105748]: Upgrade: Complete! 2024-09-21T09:56:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:46 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:56:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:46 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:56:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:46 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:56:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:46 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:56:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:46 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all grafana 2024-09-21T09:56:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:46 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all loki 2024-09-21T09:56:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:46 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all promtail 2024-09-21T09:56:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:46 smithi045 ceph-mon[33087]: Upgrade: Finalizing container_image settings 2024-09-21T09:56:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:46 smithi045 ceph-mon[33087]: Upgrade: Complete! 2024-09-21T09:56:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:47 smithi032 ceph-mon[101299]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:47 smithi032 ceph-mon[105748]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:47 smithi045 ceph-mon[33087]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:49.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:49 smithi032 ceph-mon[101299]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:49.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:49.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:49.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:56:49] "GET /metrics HTTP/1.1" 200 33799 "" "Prometheus/2.51.0" 2024-09-21T09:56:49.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:49 smithi032 ceph-mon[105748]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:49.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:49.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:49 smithi045 ceph-mon[33087]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:49 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:49 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:56:51.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:51 smithi032 ceph-mon[101299]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:51.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:51 smithi032 ceph-mon[105748]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:51 smithi045 ceph-mon[33087]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:53.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:53 smithi032 ceph-mon[101299]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:53 smithi032 ceph-mon[105748]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:53.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:53 smithi045 ceph-mon[33087]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:55.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:55 smithi032 ceph-mon[101299]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:55.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:55 smithi032 ceph-mon[105748]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:55.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:55 smithi045 ceph-mon[33087]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:57.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:57 smithi032 ceph-mon[101299]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:57.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:57 smithi032 ceph-mon[105748]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:57.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:57 smithi045 ceph-mon[33087]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:59.783 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:56:59 smithi032 ceph-mon[101299]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:59.783 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:56:59 smithi032 ceph-mon[105748]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:59.783 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:56:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:56:59] "GET /metrics HTTP/1.1" 200 33799 "" "Prometheus/2.51.0" 2024-09-21T09:56:59.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:56:59 smithi045 ceph-mon[33087]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:01.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:01 smithi045 ceph-mon[33087]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:01 smithi032 ceph-mon[101299]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:01 smithi032 ceph-mon[105748]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:03 smithi045 ceph-mon[33087]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:03 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:03 smithi032 ceph-mon[101299]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:03 smithi032 ceph-mon[105748]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:05 smithi045 ceph-mon[33087]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:05 smithi032 ceph-mon[105748]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:05 smithi032 ceph-mon[101299]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:07 smithi045 ceph-mon[33087]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:07 smithi032 ceph-mon[101299]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:07 smithi032 ceph-mon[105748]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:09.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:09 smithi045 ceph-mon[33087]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:09 smithi032 ceph-mon[105748]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:09 smithi032 ceph-mon[101299]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:09.958 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:57:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:57:09] "GET /metrics HTTP/1.1" 200 33796 "" "Prometheus/2.51.0" 2024-09-21T09:57:10.762 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'sleep 60' 2024-09-21T09:57:10.977 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:11.480 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 09:57:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=infra.usagestats t=2024-09-21T09:57:11.12883958Z level=info msg="Usage stats are ready to report" 2024-09-21T09:57:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:11 smithi032 ceph-mon[105748]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:11 smithi032 ceph-mon[105748]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:11 smithi032 ceph-mon[101299]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:11 smithi032 ceph-mon[101299]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:11.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:11 smithi045 ceph-mon[33087]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:11.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:11 smithi045 ceph-mon[33087]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:13 smithi045 ceph-mon[33087]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:13 smithi032 ceph-mon[101299]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:13 smithi032 ceph-mon[105748]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:15.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:15 smithi045 ceph-mon[33087]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:15.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:15 smithi032 ceph-mon[101299]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:15.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:15 smithi032 ceph-mon[105748]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:17.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:17 smithi045 ceph-mon[33087]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:17 smithi032 ceph-mon[101299]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:17 smithi032 ceph-mon[105748]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:18.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:18 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:18.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:18.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:19 smithi045 ceph-mon[33087]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:19 smithi032 ceph-mon[101299]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:19 smithi032 ceph-mon[105748]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:19.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:57:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:57:19] "GET /metrics HTTP/1.1" 200 33794 "" "Prometheus/2.51.0" 2024-09-21T09:57:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:21 smithi045 ceph-mon[33087]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:21.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:21 smithi032 ceph-mon[101299]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:21 smithi032 ceph-mon[105748]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:23 smithi045 ceph-mon[33087]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:23 smithi032 ceph-mon[101299]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:23.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:23 smithi032 ceph-mon[105748]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:25 smithi045 ceph-mon[33087]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:25.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:25 smithi032 ceph-mon[101299]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:25.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:25 smithi032 ceph-mon[105748]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:27 smithi045 ceph-mon[33087]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:27 smithi032 ceph-mon[101299]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:27 smithi032 ceph-mon[105748]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:29.759 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:29 smithi032 ceph-mon[105748]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:29.759 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:29 smithi032 ceph-mon[101299]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:29.760 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:57:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:57:29] "GET /metrics HTTP/1.1" 200 33794 "" "Prometheus/2.51.0" 2024-09-21T09:57:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:29 smithi045 ceph-mon[33087]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:31.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:31 smithi045 ceph-mon[33087]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:31.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:31 smithi032 ceph-mon[101299]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:31 smithi032 ceph-mon[105748]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:33.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:33 smithi045 ceph-mon[33087]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:33.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:33 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:33.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:33 smithi032 ceph-mon[101299]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:33 smithi032 ceph-mon[105748]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:35.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:35 smithi045 ceph-mon[33087]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:35 smithi032 ceph-mon[101299]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:35 smithi032 ceph-mon[105748]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:37.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:37 smithi045 ceph-mon[33087]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:37.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:37 smithi032 ceph-mon[101299]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:37.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:37 smithi032 ceph-mon[105748]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:39 smithi045 ceph-mon[33087]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:39.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:39 smithi032 ceph-mon[101299]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:39.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:39 smithi032 ceph-mon[105748]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:39.962 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:57:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:57:39] "GET /metrics HTTP/1.1" 200 33799 "" "Prometheus/2.51.0" 2024-09-21T09:57:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:41 smithi045 ceph-mon[33087]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:41.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:41 smithi032 ceph-mon[105748]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:41 smithi032 ceph-mon[101299]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:43 smithi045 ceph-mon[33087]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:43.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:43 smithi032 ceph-mon[101299]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:43.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:43 smithi032 ceph-mon[105748]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:45 smithi045 ceph-mon[33087]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:45 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:45.937 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:45 smithi032 ceph-mon[105748]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:45.937 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:45.937 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:45 smithi032 ceph-mon[101299]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:45.937 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:47 smithi045 ceph-mon[33087]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:47.807 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:47 smithi032 ceph-mon[101299]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:47.808 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:47 smithi032 ceph-mon[105748]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:48.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:48 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:48.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:48.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:57:49] "GET /metrics HTTP/1.1" 200 33799 "" "Prometheus/2.51.0" 2024-09-21T09:57:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[101299]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[105748]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:49.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:49.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:49 smithi045 ceph-mon[33087]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:57:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:49 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:49 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:49 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:49 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:50 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:50 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:50 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:51.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:57:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:51 smithi032 ceph-mon[101299]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:51 smithi032 ceph-mon[101299]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:57:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:51 smithi032 ceph-mon[105748]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:51 smithi032 ceph-mon[105748]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:57:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:51 smithi045 ceph-mon[33087]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:51 smithi045 ceph-mon[33087]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:57:53.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:52 smithi032 ceph-mon[101299]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:52 smithi032 ceph-mon[105748]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:53.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:52 smithi045 ceph-mon[33087]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:55.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:55 smithi032 ceph-mon[101299]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:55.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:55 smithi032 ceph-mon[105748]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:55.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:55 smithi045 ceph-mon[33087]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:57.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:57 smithi032 ceph-mon[105748]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:57.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:57 smithi032 ceph-mon[101299]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:57.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:57 smithi045 ceph-mon[33087]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:59.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:57:59 smithi032 ceph-mon[101299]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:59.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:57:59 smithi032 ceph-mon[105748]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:59.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:57:59 smithi045 ceph-mon[33087]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:57:59.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:57:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:57:59] "GET /metrics HTTP/1.1" 200 33799 "" "Prometheus/2.51.0" 2024-09-21T09:58:01.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:01 smithi032 ceph-mon[101299]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:58:01.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:01 smithi032 ceph-mon[105748]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:58:01.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:01 smithi045 ceph-mon[33087]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:58:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:03 smithi032 ceph-mon[105748]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:03.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:03 smithi032 ceph-mon[101299]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:03.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:03 smithi045 ceph-mon[33087]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:04.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:04 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:05.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:05 smithi032 ceph-mon[105748]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:05.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:05 smithi032 ceph-mon[101299]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:05.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:05 smithi045 ceph-mon[33087]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:07.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:07 smithi032 ceph-mon[105748]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:07 smithi032 ceph-mon[101299]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:07 smithi045 ceph-mon[33087]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:09.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:09 smithi032 ceph-mon[101299]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:09 smithi032 ceph-mon[105748]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:09 smithi045 ceph-mon[33087]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:58:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:58:09] "GET /metrics HTTP/1.1" 200 33798 "" "Prometheus/2.51.0" 2024-09-21T09:58:11.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:11 smithi032 ceph-mon[101299]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:11.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:11 smithi032 ceph-mon[105748]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:11 smithi045 ceph-mon[33087]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:13.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:13 smithi032 ceph-mon[101299]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:13 smithi032 ceph-mon[105748]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:13.327 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-21T09:58:13.356 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:13 smithi045 ceph-mon[33087]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:13.544 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:15.184 INFO:teuthology.orchestra.run.smithi032.stdout:wait for majority of mons to upgrade 2024-09-21T09:58:15.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:15 smithi032 ceph-mon[101299]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:15.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:15 smithi032 ceph-mon[105748]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:15 smithi045 ceph-mon[33087]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:15.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-21T09:58:16.061 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:17.398 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:17 smithi032 ceph-mon[101299]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:17.398 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:17 smithi032 ceph-mon[105748]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:17 smithi045 ceph-mon[33087]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:19.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:19 smithi032 ceph-mon[101299]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:19.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:19 smithi032 ceph-mon[101299]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:19.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:19.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:19 smithi032 ceph-mon[105748]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:19.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:19 smithi032 ceph-mon[105748]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:19.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:19 smithi045 ceph-mon[33087]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:19 smithi045 ceph-mon[33087]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:19 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:19.789 INFO:teuthology.orchestra.run.smithi032.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:58:19.790 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:58:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:58:19] "GET /metrics HTTP/1.1" 200 33795 "" "Prometheus/2.51.0" 2024-09-21T09:58:20.429 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:58:20.655 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[105748]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[101299]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:21.029 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:21.030 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:21.030 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:20 smithi045 ceph-mon[33087]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:58:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:20 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:20 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:20 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:20 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:20 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:21.929 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:21 smithi032 ceph-mon[105748]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:58:21.930 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:21 smithi032 ceph-mon[105748]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:21.930 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:21 smithi032 ceph-mon[101299]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:58:21.930 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:21 smithi032 ceph-mon[101299]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:21 smithi045 ceph-mon[33087]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:58:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:21 smithi045 ceph-mon[33087]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:23.740 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:58:24.115 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:23 smithi045 ceph-mon[33087]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:24.116 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:23 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:24.116 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:23 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:24.116 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:23 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:24.116 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:23 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:24.116 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:23 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:58:24.116 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:23 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:58:24.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[105748]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[101299]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:24.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:24.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:58:24.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (3m) 35s ago 9m 20.0M - 0.27.0 11f11916f8cd d6fa4f2dece8 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (3m) 34s ago 8m 75.9M - 10.4.8 5aad1d7cf004 0ad08e43d776 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (7m) 35s ago 7m 18.4M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (7m) 34s ago 7m 17.9M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (4m) 34s ago 12m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (4m) 35s ago 14m 509M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (2m) 35s ago 14m 40.1M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 525e81f3c17e 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (13m) 34s ago 13m 56.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 071e4635dabc 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (112s) 35s ago 13m 32.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 233d45e2730d 2024-09-21T09:58:24.332 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (4m) 35s ago 8m 10.0M - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (4m) 34s ago 8m 10.6M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (12m) 35s ago 12m 59.3M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (12m) 35s ago 12m 53.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (11m) 35s ago 11m 58.1M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (11m) 35s ago 11m 46.2M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (11m) 34s ago 11m 54.2M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (10m) 34s ago 10m 48.8M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (10m) 34s ago 10m 47.9M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (9m) 34s ago 9m 50.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:58:24.333 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (3m) 34s ago 9m 45.4M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 1, 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:58:24.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:58:24.689 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:58:24.689 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:58:24.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:58:24.689 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:58:24.689 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 11, 2024-09-21T09:58:24.689 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:58:24.689 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:58:24.689 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:58:24.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all mgr 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3071649601' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:24.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:58:24.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:58:24.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all mgr 2024-09-21T09:58:24.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:58:24.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:24.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3071649601' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:24.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:24.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:24.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:24.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:24 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:24.996 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:58:24.997 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:58:24.997 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:58:24.997 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-21T09:58:24.997 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T09:58:24.997 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "2/3 daemons upgraded", 2024-09-21T09:58:24.997 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:58:24.997 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:58:24.997 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:58:25.074 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:58:25.074 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:58:25.074 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: Upgrade: Setting container_image for all mgr 2024-09-21T09:58:25.074 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:58:25.074 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:25.074 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: from='client.? 172.21.15.32:0/3071649601' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:25.074 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:25.074 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:25.075 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:25.075 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:24 smithi045 ceph-mon[33087]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:26.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:25 smithi045 ceph-mon[33087]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:26.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:25 smithi045 ceph-mon[33087]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:26.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:25 smithi045 ceph-mon[33087]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:26.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:25 smithi045 ceph-mon[33087]: Upgrade: Updating mon.b 2024-09-21T09:58:26.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:25 smithi045 ceph-mon[33087]: Deploying daemon mon.b on smithi045 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[105748]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[105748]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[105748]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[105748]: Upgrade: Updating mon.b 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[105748]: Deploying daemon mon.b on smithi045 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[101299]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[101299]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[101299]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[101299]: Upgrade: Updating mon.b 2024-09-21T09:58:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:25 smithi032 ceph-mon[101299]: Deploying daemon mon.b on smithi045 2024-09-21T09:58:26.453 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:26 smithi045 systemd[1]: Stopping Ceph mon.b for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:58:26.783 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b[33083]: 2024-09-21T09:58:26.699+0000 7f0013ac3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:58:26.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b[33083]: 2024-09-21T09:58:26.699+0000 7f0013ac3640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:58:27.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:26 smithi045 podman[95817]: 2024-09-21 09:58:26.781370609 +0000 UTC m=+0.181018503 container died 071e4635dabcea783b4a4a5389b6195dfc96fdc4977316509d545bdf36ced58b (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=reef-6a1bdf8, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T09:58:27.410 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:27 smithi045 podman[95817]: 2024-09-21 09:58:27.157402265 +0000 UTC m=+0.557050156 container cleanup 071e4635dabcea783b4a4a5389b6195dfc96fdc4977316509d545bdf36ced58b (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b, ceph=True, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T09:58:27.410 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:27 smithi045 bash[95817]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b 2024-09-21T09:58:27.748 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:27 smithi045 podman[95830]: 2024-09-21 09:58:27.408787078 +0000 UTC m=+0.629306248 container remove 071e4635dabcea783b4a4a5389b6195dfc96fdc4977316509d545bdf36ced58b (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:58:27.748 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:27 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.b.service: Deactivated successfully. 2024-09-21T09:58:27.748 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:27 smithi045 systemd[1]: Stopped Ceph mon.b for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:58:27.748 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:27 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.b.service: Consumed 10.297s CPU time. 2024-09-21T09:58:28.002 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:27 smithi045 systemd[1]: Starting Ceph mon.b for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:58:28.003 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:27 smithi045 podman[95957]: 2024-09-21 09:58:27.86051034 +0000 UTC m=+0.016465574 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:58:28.003 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 podman[95957]: 2024-09-21 09:58:28.001348676 +0000 UTC m=+0.157303901 container create 7115dfb7957aa655f8d4a64b56d1b2b4277b6b6bebdae068d37f17a0744714c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2) 2024-09-21T09:58:28.273 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 podman[95957]: 2024-09-21 09:58:28.245751825 +0000 UTC m=+0.401707056 container init 7115dfb7957aa655f8d4a64b56d1b2b4277b6b6bebdae068d37f17a0744714c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:58:28.273 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 podman[95957]: 2024-09-21 09:58:28.249036505 +0000 UTC m=+0.404991730 container start 7115dfb7957aa655f8d4a64b56d1b2b4277b6b6bebdae068d37f17a0744714c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T09:58:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:58:28.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 2 2024-09-21T09:58:28.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: pidfile_write: ignore empty --pid-file 2024-09-21T09:58:28.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: load: jerasure load: lrc 2024-09-21T09:58:28.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:58:28.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Git sha 0 2024-09-21T09:58:28.537 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:58:28.537 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: DB SUMMARY 2024-09-21T09:58:28.537 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: DB Session ID: 2GSUKH1U5OOZNQ0IFQGZ 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1404 Bytes 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 7655044 ; 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.env: 0x55ec4c4ea120 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.info_log: 0x55ec4da23700 2024-09-21T09:58:28.538 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.statistics: (nil) 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.use_fsync: 0 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.db_log_dir: 2024-09-21T09:58:28.539 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.wal_dir: 2024-09-21T09:58:28.540 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.write_buffer_manager: 0x55ec4da27b80 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:58:28.541 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.unordered_write: 0 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.row_cache: None 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.wal_filter: None 2024-09-21T09:58:28.542 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.wal_compression: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:58:28.543 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_open_files: -1 2024-09-21T09:58:28.544 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Compression algorithms supported: 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: kZSTD supported: 0 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:58:28.545 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.merge_operator: 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_filter: None 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ec4da23320) 2024-09-21T09:58:28.546 INFO:journalctl@ceph.mon.b.smithi045.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: index_type: 0 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: data_block_index_type: 0 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: index_shortening: 1 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: checksum: 4 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: no_block_cache: 0 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_cache: 0x55ec4da17610 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_cache_options: 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: capacity : 536870912 2024-09-21T09:58:28.547 INFO:journalctl@ceph.mon.b.smithi045.stdout: num_shard_bits : 4 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: strict_capacity_limit : 0 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_cache_compressed: (nil) 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: persistent_cache: (nil) 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_size: 4096 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_size_deviation: 10 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_restart_interval: 16 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: index_block_restart_interval: 1 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: metadata_block_size: 4096 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: partition_filters: 0 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: use_delta_encoding: 1 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: filter_policy: bloomfilter 2024-09-21T09:58:28.548 INFO:journalctl@ceph.mon.b.smithi045.stdout: whole_key_filtering: 1 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout: verify_compression: 0 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout: format_version: 5 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout: enable_index_compression: 1 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_align: 0 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout: max_auto_readahead_size: 262144 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout: prepopulate_block_cache: 0 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression: NoCompression 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:58:28.549 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.num_levels: 7 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:58:28.550 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:58:28.551 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:58:28.552 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:58:28.553 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.ttl: 2592000 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:58:28.554 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 12992, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: bb274b07-2c1a-4ae0-af06-22e845856a18 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912708276468, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912708294955, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4686585, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12997, "largest_seqno": 16635, "table_properties": {"data_size": 4672549, "index_size": 8957, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4037, "raw_key_size": 35565, "raw_average_key_size": 22, "raw_value_size": 4640916, "raw_average_value_size": 2916, "num_data_blocks": 404, "num_entries": 1591, "num_filter_entries": 1591, "num_deletions": 261, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726912708, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "bb274b07-2c1a-4ae0-af06-22e845856a18", "db_session_id": "2GSUKH1U5OOZNQ0IFQGZ", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912708295086, "job": 1, "event": "recovery_finished"} 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-21T09:58:28.555 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ec4da46e00 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: DB pointer 0x55ec4db48000 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: ** DB Stats ** 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2024-09-21T09:58:28.556 INFO:journalctl@ceph.mon.b.smithi045.stdout: ** Compaction Stats [default] ** 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: L0 1/0 4.47 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 659.6 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: L6 1/0 7.80 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: Sum 2/0 12.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 659.6 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 659.6 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: ** Compaction Stats [default] ** 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 659.6 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2024-09-21T09:58:28.557 INFO:journalctl@ceph.mon.b.smithi045.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: Cumulative compaction: 0.00 GB write, 208.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: Interval compaction: 0.00 GB write, 208.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: Block cache BinnedLRUCache@0x55ec4da17610#2 capacity: 512.00 MB usage: 13.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,9.08 KB,0.00173151%) Misc(1,0.00 KB,0%) 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: starting mon.b rank 1 at public addrs [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] at bind addrs [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:58:28.558 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: mon.b@-1(???) e3 preinit fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: mon.b@-1(???).mds e9 new map 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: mon.b@-1(???).mds e9 print_map 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout: e9 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout: legacy client fscid: 1 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout: Filesystem 'foo' (1) 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout: fs_name foo 2024-09-21T09:58:28.559 INFO:journalctl@ceph.mon.b.smithi045.stdout: epoch 8 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: created 2024-09-21T09:50:22.390699+0000 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: modified 2024-09-21T09:50:36.781550+0000 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: tableserver 0 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: root 0 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: session_timeout 60 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: session_autoclose 300 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: max_file_size 1099511627776 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: max_xattr_size 65536 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: required_client_features {} 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: last_failure 0 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: last_failure_osd_epoch 0 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:58:28.560 INFO:journalctl@ceph.mon.b.smithi045.stdout: max_mds 1 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: in 0 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: up {0=14655} 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: failed 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: damaged 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: stopped 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: data_pools [4] 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: metadata_pool 3 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: inline_data disabled 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: balancer 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: bal_rank_mask -1 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: standby_count_wanted 1 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: [mds.foo.smithi032.fdolcm{0:14655} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.32:6834/2708717542,v1:172.21.15.32:6835/2708717542] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2024-09-21T09:58:28.561 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout: Standby daemons: 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout: [mds.foo.smithi045.jnfsfb{-1:24496} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.45:6832/3743591142,v1:172.21.15.45:6833/3743591142] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: mon.b@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 ceph-mon[95976]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 bash[95957]: 7115dfb7957aa655f8d4a64b56d1b2b4277b6b6bebdae068d37f17a0744714c2 2024-09-21T09:58:28.562 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:28 smithi045 systemd[1]: Started Ceph mon.b for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:58:29.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:58:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:58:29] "GET /metrics HTTP/1.1" 200 33795 "" "Prometheus/2.51.0" 2024-09-21T09:58:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:58:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:58:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: mgrmap e40: y(active, since 2m), standbys: x 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: overall HEALTH_OK 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: mon.b calling monitor election 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: mon.c calling monitor election 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: mon.a calling monitor election 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: monmap epoch 4 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: last_changed 2024-09-21T09:58:29.745293+0000 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: created 2024-09-21T09:43:34.453975+0000 2024-09-21T09:58:31.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: min_mon_release 19 (squid) 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: election_strategy: 1 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: 1: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.b 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: mgrmap e40: y(active, since 2m), standbys: x 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: overall HEALTH_OK 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:31.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:31.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:58:31.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:58:31.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:58:31.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: mgrmap e40: y(active, since 2m), standbys: x 2024-09-21T09:58:31.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: overall HEALTH_OK 2024-09-21T09:58:31.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: mon.b calling monitor election 2024-09-21T09:58:31.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: mon.c calling monitor election 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: mon.a calling monitor election 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: monmap epoch 4 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: last_changed 2024-09-21T09:58:29.745293+0000 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: created 2024-09-21T09:43:34.453975+0000 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: min_mon_release 19 (squid) 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: election_strategy: 1 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: 1: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.b 2024-09-21T09:58:31.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: mgrmap e40: y(active, since 2m), standbys: x 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: overall HEALTH_OK 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: mgrmap e40: y(active, since 2m), standbys: x 2024-09-21T09:58:31.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: overall HEALTH_OK 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: mon.b calling monitor election 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: mon.c calling monitor election 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: mon.a calling monitor election 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: monmap epoch 4 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: last_changed 2024-09-21T09:58:29.745293+0000 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: created 2024-09-21T09:43:34.453975+0000 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: min_mon_release 19 (squid) 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: election_strategy: 1 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: 0: [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon.a 2024-09-21T09:58:31.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: 1: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.b 2024-09-21T09:58:31.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: 2: [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0] mon.c 2024-09-21T09:58:31.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: fsmap foo:1 {0=foo.smithi032.fdolcm=up:active} 1 up:standby 2024-09-21T09:58:31.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:58:31.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: mgrmap e40: y(active, since 2m), standbys: x 2024-09-21T09:58:31.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: overall HEALTH_OK 2024-09-21T09:58:31.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:31.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:31.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:31.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:32.030 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:31 smithi045 ceph-mon[95976]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:31 smithi032 ceph-mon[101299]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:31 smithi032 ceph-mon[105748]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:33.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:58:33 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T09:58:33.306+0000 7fce75a2e640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:58:34.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:33 smithi045 ceph-mon[95976]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:34.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:33 smithi032 ceph-mon[105748]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:33 smithi032 ceph-mon[101299]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:35 smithi045 ceph-mon[95976]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:35 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:35 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:35.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:35 smithi032 ceph-mon[105748]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:35.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:35 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:35.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:35 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:35.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:35 smithi032 ceph-mon[101299]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:35.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:35 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:35.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:35 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:37.423 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:37 smithi045 ceph-mon[95976]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:37.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:37 smithi032 ceph-mon[101299]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:37.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:37 smithi032 ceph-mon[105748]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:58:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:38.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:38 smithi045 ceph-mon[95976]: Reconfiguring daemon mon.a on smithi032 2024-09-21T09:58:38.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:38.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:38.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:38.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:38.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:38.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:58:38.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:38.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:38.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:38.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[101299]: Reconfiguring daemon mon.a on smithi032 2024-09-21T09:58:38.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:38.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:38.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:38.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:38.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:38.873 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:58:38.873 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:38.873 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:38.873 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:38.873 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:38 smithi032 ceph-mon[105748]: Reconfiguring daemon mon.a on smithi032 2024-09-21T09:58:39.696 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:58:39] "GET /metrics HTTP/1.1" 200 33808 "" "Prometheus/2.51.0" 2024-09-21T09:58:39.697 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[105748]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:39.697 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:39.697 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:39.697 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:58:39.697 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:58:39.697 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:39.697 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[101299]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:39.697 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:39.697 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:39.698 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:58:39.698 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:58:39.698 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:39 smithi045 ceph-mon[95976]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:58:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:58:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:40.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[101299]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:58:40.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[101299]: Reconfiguring daemon mgr.y on smithi032 2024-09-21T09:58:40.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:40.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:40.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:40.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:40.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:40.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[105748]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:58:40.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[105748]: Reconfiguring daemon mgr.y on smithi032 2024-09-21T09:58:40.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:40.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:40.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:40.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:40.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:40 smithi045 ceph-mon[95976]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:58:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:40 smithi045 ceph-mon[95976]: Reconfiguring daemon mgr.y on smithi032 2024-09-21T09:58:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:41 smithi045 ceph-mon[95976]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:41 smithi045 ceph-mon[95976]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:58:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:41 smithi045 ceph-mon[95976]: Reconfiguring daemon mon.c on smithi032 2024-09-21T09:58:41.841 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:41 smithi032 ceph-mon[101299]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:41.842 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:41 smithi032 ceph-mon[101299]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:58:41.842 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:41 smithi032 ceph-mon[101299]: Reconfiguring daemon mon.c on smithi032 2024-09-21T09:58:41.842 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:41 smithi032 ceph-mon[105748]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:41.842 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:41 smithi032 ceph-mon[105748]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:58:41.842 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:41 smithi032 ceph-mon[105748]: Reconfiguring daemon mon.c on smithi032 2024-09-21T09:58:43.190 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:43.190 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:43.190 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[105748]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:58:43.190 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:58:43.190 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:43.190 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[105748]: Reconfiguring daemon osd.0 on smithi032 2024-09-21T09:58:43.191 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[105748]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:43.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:43.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:43.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[101299]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:58:43.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:58:43.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:43.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[101299]: Reconfiguring daemon osd.0 on smithi032 2024-09-21T09:58:43.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:42 smithi032 ceph-mon[101299]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:42 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:42 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:42 smithi045 ceph-mon[95976]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:58:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:42 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:58:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:42 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:42 smithi045 ceph-mon[95976]: Reconfiguring daemon osd.0 on smithi032 2024-09-21T09:58:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:42 smithi045 ceph-mon[95976]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:44.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:44 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:44.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:44 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:44.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:44 smithi045 ceph-mon[95976]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:58:44.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:44 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:58:44.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:44 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:44.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:44 smithi045 ceph-mon[95976]: Reconfiguring daemon osd.1 on smithi032 2024-09-21T09:58:44.673 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:44.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:44.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[105748]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:58:44.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:58:44.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:44.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[105748]: Reconfiguring daemon osd.1 on smithi032 2024-09-21T09:58:44.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:44.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:44.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[101299]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:58:44.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:58:44.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:44.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:44 smithi032 ceph-mon[101299]: Reconfiguring daemon osd.1 on smithi032 2024-09-21T09:58:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:45 smithi045 ceph-mon[95976]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:45 smithi045 ceph-mon[95976]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:58:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:58:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:45 smithi045 ceph-mon[95976]: Reconfiguring daemon osd.2 on smithi032 2024-09-21T09:58:46.122 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[105748]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:46.123 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:46.123 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:46.123 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[105748]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:58:46.123 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:58:46.123 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:46.123 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[105748]: Reconfiguring daemon osd.2 on smithi032 2024-09-21T09:58:46.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[101299]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:46.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:46.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:46.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[101299]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:58:46.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:58:46.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:46.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:45 smithi032 ceph-mon[101299]: Reconfiguring daemon osd.2 on smithi032 2024-09-21T09:58:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:47 smithi045 ceph-mon[95976]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:47 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:47 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:47 smithi045 ceph-mon[95976]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:58:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:47 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:58:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:47 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:47 smithi045 ceph-mon[95976]: Reconfiguring daemon osd.3 on smithi032 2024-09-21T09:58:47.638 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[101299]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:47.638 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:47.638 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:47.638 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[101299]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:58:47.638 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:58:47.638 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:47.638 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[101299]: Reconfiguring daemon osd.3 on smithi032 2024-09-21T09:58:47.639 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[105748]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:47.639 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:47.639 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:47.639 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[105748]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:58:47.639 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:58:47.639 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:47.639 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:47 smithi032 ceph-mon[105748]: Reconfiguring daemon osd.3 on smithi032 2024-09-21T09:58:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:48 smithi045 ceph-mon[95976]: Reconfiguring mds.foo.smithi032.fdolcm (monmap changed)... 2024-09-21T09:58:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi032.fdolcm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:58:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:48 smithi045 ceph-mon[95976]: Reconfiguring daemon mds.foo.smithi032.fdolcm on smithi032 2024-09-21T09:58:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:49.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[101299]: Reconfiguring mds.foo.smithi032.fdolcm (monmap changed)... 2024-09-21T09:58:49.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi032.fdolcm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:58:49.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:49.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[101299]: Reconfiguring daemon mds.foo.smithi032.fdolcm on smithi032 2024-09-21T09:58:49.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:49.085 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.085 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.085 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[105748]: Reconfiguring mds.foo.smithi032.fdolcm (monmap changed)... 2024-09-21T09:58:49.085 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi032.fdolcm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:58:49.085 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:49.085 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[105748]: Reconfiguring daemon mds.foo.smithi032.fdolcm on smithi032 2024-09-21T09:58:49.085 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:49.884 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:49 smithi045 ceph-mon[95976]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:49.885 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.885 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.885 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:49.885 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:49.885 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:49.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:58:49] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.51.0" 2024-09-21T09:58:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[101299]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[105748]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:58:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:58:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:51.031 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:50 smithi045 ceph-mon[95976]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:58:51.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:50 smithi045 ceph-mon[95976]: Reconfiguring daemon mon.b on smithi045 2024-09-21T09:58:51.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:51.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:51.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:58:51.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:58:51.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[101299]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:58:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[101299]: Reconfiguring daemon mon.b on smithi045 2024-09-21T09:58:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:58:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:58:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[105748]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:58:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[105748]: Reconfiguring daemon mon.b on smithi045 2024-09-21T09:58:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:58:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:58:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:51 smithi045 ceph-mon[95976]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:51 smithi045 ceph-mon[95976]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:58:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:51 smithi045 ceph-mon[95976]: Reconfiguring daemon mgr.x on smithi045 2024-09-21T09:58:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:51 smithi032 ceph-mon[101299]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:51 smithi032 ceph-mon[101299]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:58:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:51 smithi032 ceph-mon[101299]: Reconfiguring daemon mgr.x on smithi045 2024-09-21T09:58:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:51 smithi032 ceph-mon[105748]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:51 smithi032 ceph-mon[105748]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:58:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:51 smithi032 ceph-mon[105748]: Reconfiguring daemon mgr.x on smithi045 2024-09-21T09:58:53.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[101299]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:53.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:53.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:53.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[101299]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:58:53.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:58:53.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:53.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[101299]: Reconfiguring daemon osd.4 on smithi045 2024-09-21T09:58:53.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[105748]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:53.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:53.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:53.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[105748]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:58:53.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:58:53.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:53.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:53 smithi032 ceph-mon[105748]: Reconfiguring daemon osd.4 on smithi045 2024-09-21T09:58:53.496 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:53 smithi045 ceph-mon[95976]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:53.496 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:53.496 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:53.496 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:53 smithi045 ceph-mon[95976]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:58:53.496 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:58:53.496 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:53.496 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:53 smithi045 ceph-mon[95976]: Reconfiguring daemon osd.4 on smithi045 2024-09-21T09:58:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[105748]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:58:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:58:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:54.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[105748]: Reconfiguring daemon osd.5 on smithi045 2024-09-21T09:58:54.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:54.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:54.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[101299]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:58:54.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:58:54.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:54.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:54 smithi032 ceph-mon[101299]: Reconfiguring daemon osd.5 on smithi045 2024-09-21T09:58:55.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:55.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:55.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:54 smithi045 ceph-mon[95976]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:58:55.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:58:55.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:55.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:54 smithi045 ceph-mon[95976]: Reconfiguring daemon osd.5 on smithi045 2024-09-21T09:58:55.327 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T09:58:55.892 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:55 smithi045 ceph-mon[95976]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:55.892 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:55.892 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:55.892 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:58:55.893 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:55.921 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:58:55.921 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (3m) 67s ago 9m 20.0M - 0.27.0 11f11916f8cd d6fa4f2dece8 2024-09-21T09:58:55.921 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (3m) 22s ago 9m 78.1M - 10.4.8 5aad1d7cf004 0ad08e43d776 2024-09-21T09:58:55.921 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (8m) 67s ago 8m 18.4M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T09:58:55.921 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (8m) 22s ago 8m 18.4M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T09:58:55.922 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (4m) 22s ago 13m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T09:58:55.922 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (5m) 67s ago 15m 509M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T09:58:55.922 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (2m) 67s ago 15m 40.1M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 525e81f3c17e 2024-09-21T09:58:55.922 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (28s) 22s ago 13m 27.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7115dfb7957a 2024-09-21T09:58:55.922 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (2m) 67s ago 13m 32.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 233d45e2730d 2024-09-21T09:58:55.922 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (4m) 67s ago 9m 10.0M - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T09:58:55.923 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (4m) 22s ago 9m 10.1M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T09:58:55.923 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (13m) 67s ago 13m 59.3M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T09:58:55.923 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (12m) 67s ago 12m 53.4M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T09:58:55.923 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (12m) 67s ago 12m 58.1M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T09:58:55.923 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (12m) 67s ago 12m 46.2M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T09:58:55.923 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (11m) 22s ago 11m 54.9M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T09:58:55.924 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (11m) 22s ago 11m 49.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T09:58:55.924 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (10m) 22s ago 10m 48.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T09:58:55.924 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (10m) 22s ago 10m 51.2M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T09:58:55.924 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (4m) 22s ago 9m 45.4M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T09:58:55.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[105748]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:55.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:55.925 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:55.925 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:58:55.925 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:55.925 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[101299]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:55.925 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:55.925 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:55.925 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:58:55.926 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:56.281 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:58:56.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T09:58:56.281 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:58:56.281 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 10, 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T09:58:56.282 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:58:56.587 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [ 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout: "mon" 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "3/3 daemons upgraded", 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T09:58:56.588 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T09:58:56.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[105748]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:58:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[105748]: Reconfiguring daemon osd.6 on smithi045 2024-09-21T09:58:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[105748]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[105748]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/146072668' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[101299]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:58:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[101299]: Reconfiguring daemon osd.6 on smithi045 2024-09-21T09:58:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[101299]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[101299]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/146072668' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:56 smithi045 ceph-mon[95976]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:58:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:56 smithi045 ceph-mon[95976]: Reconfiguring daemon osd.6 on smithi045 2024-09-21T09:58:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:56 smithi045 ceph-mon[95976]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:56 smithi045 ceph-mon[95976]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/146072668' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[101299]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[101299]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[101299]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[101299]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[101299]: Reconfiguring daemon osd.7 on smithi045 2024-09-21T09:58:57.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[105748]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:57.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[105748]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:57.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[105748]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:57.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:57.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:57.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[105748]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:58:57.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:58:57.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:57.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:57 smithi032 ceph-mon[105748]: Reconfiguring daemon osd.7 on smithi045 2024-09-21T09:58:57.994 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:57 smithi045 ceph-mon[95976]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:57.994 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:57 smithi045 ceph-mon[95976]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:57.994 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:57 smithi045 ceph-mon[95976]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:57.994 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:57 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:57.994 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:57 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:57.994 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:57 smithi045 ceph-mon[95976]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:58:57.994 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:57 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:58:57.995 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:57 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:57.995 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:57 smithi045 ceph-mon[95976]: Reconfiguring daemon osd.7 on smithi045 2024-09-21T09:58:59.417 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:59 smithi045 ceph-mon[95976]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:59.418 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:59 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:59.418 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:59 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:59.418 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:59 smithi045 ceph-mon[95976]: Reconfiguring mds.foo.smithi045.jnfsfb (monmap changed)... 2024-09-21T09:58:59.418 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:59 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi045.jnfsfb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:58:59.418 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:59 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:59.418 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:58:59 smithi045 ceph-mon[95976]: Reconfiguring daemon mds.foo.smithi045.jnfsfb on smithi045 2024-09-21T09:58:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[101299]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[101299]: Reconfiguring mds.foo.smithi045.jnfsfb (monmap changed)... 2024-09-21T09:58:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi045.jnfsfb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:58:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[101299]: Reconfiguring daemon mds.foo.smithi045.jnfsfb on smithi045 2024-09-21T09:58:59.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[105748]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:58:59.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:59.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:58:59.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[105748]: Reconfiguring mds.foo.smithi045.jnfsfb (monmap changed)... 2024-09-21T09:58:59.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi045.jnfsfb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:58:59.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:59.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-mon[105748]: Reconfiguring daemon mds.foo.smithi045.jnfsfb on smithi045 2024-09-21T09:58:59.757 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:58:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:58:59] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.51.0" 2024-09-21T09:59:00.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all grafana 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all loki 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all promtail 2024-09-21T09:59:00.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: Upgrade: Finalizing container_image settings 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:59:00.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: Upgrade: Complete! 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:59:00.789 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:00 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:00.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:00.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:59:00.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all grafana 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all loki 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all promtail 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: Upgrade: Finalizing container_image settings 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:59:00.827 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all grafana 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all loki 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all promtail 2024-09-21T09:59:00.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: Upgrade: Finalizing container_image settings 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:59:00.830 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: Upgrade: Complete! 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: Upgrade: Complete! 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:59:00.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:00 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:01.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:01 smithi032 ceph-mon[101299]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:01 smithi032 ceph-mon[105748]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:01 smithi045 ceph-mon[95976]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:03.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[105748]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[101299]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:03 smithi045 ceph-mon[95976]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:04.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:04.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:04.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:04.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:04.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:04.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:04.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:04.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:04.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:04.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:04.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:04.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:04.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:04.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:05.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:05.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:05.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:05.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:05.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:05.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:05.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T09:59:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:05 smithi032 ceph-mon[101299]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:05 smithi032 ceph-mon[105748]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:05 smithi045 ceph-mon[95976]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:07 smithi032 ceph-mon[101299]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:07 smithi032 ceph-mon[105748]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:07 smithi045 ceph-mon[95976]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:09 smithi032 ceph-mon[101299]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:09 smithi032 ceph-mon[105748]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:59:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:59:09] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.51.0" 2024-09-21T09:59:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:09 smithi045 ceph-mon[95976]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:11.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:11 smithi032 ceph-mon[101299]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:11.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:11 smithi032 ceph-mon[105748]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:11 smithi045 ceph-mon[95976]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:13 smithi032 ceph-mon[105748]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:13 smithi032 ceph-mon[101299]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:13 smithi045 ceph-mon[95976]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:15.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:15 smithi032 ceph-mon[105748]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:15.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:15 smithi032 ceph-mon[101299]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:15 smithi045 ceph-mon[95976]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:17 smithi032 ceph-mon[101299]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:17 smithi032 ceph-mon[105748]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:17 smithi045 ceph-mon[95976]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:18.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:19 smithi032 ceph-mon[101299]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:19 smithi032 ceph-mon[105748]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:19.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:59:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:59:19] "GET /metrics HTTP/1.1" 200 33807 "" "Prometheus/2.51.0" 2024-09-21T09:59:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:19 smithi045 ceph-mon[95976]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:21.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:21 smithi032 ceph-mon[101299]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:21 smithi032 ceph-mon[105748]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:21 smithi045 ceph-mon[95976]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:23.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:23 smithi032 ceph-mon[101299]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:23.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:23 smithi032 ceph-mon[105748]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:23 smithi045 ceph-mon[95976]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:25.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:25 smithi032 ceph-mon[101299]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:25.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:25 smithi032 ceph-mon[105748]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:26.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:25 smithi045 ceph-mon[95976]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:27.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'sleep 60' 2024-09-21T09:59:27.827 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:27.948 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:27 smithi032 ceph-mon[105748]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:27.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:27 smithi032 ceph-mon[101299]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:27 smithi045 ceph-mon[95976]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:28.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:28 smithi032 ceph-mon[101299]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:28.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:28 smithi032 ceph-mon[105748]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:28 smithi045 ceph-mon[95976]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:29.636 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:59:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:59:29] "GET /metrics HTTP/1.1" 200 33807 "" "Prometheus/2.51.0" 2024-09-21T09:59:29.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:29 smithi032 ceph-mon[101299]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:29.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:29 smithi032 ceph-mon[105748]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:29 smithi045 ceph-mon[95976]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:31.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:31 smithi032 ceph-mon[101299]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:31 smithi032 ceph-mon[105748]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:32.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:31 smithi045 ceph-mon[95976]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:33 smithi032 ceph-mon[101299]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:33 smithi032 ceph-mon[105748]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:33 smithi045 ceph-mon[95976]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:35.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:35 smithi032 ceph-mon[101299]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:35 smithi032 ceph-mon[105748]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:36.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:35 smithi045 ceph-mon[95976]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:37.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:37 smithi032 ceph-mon[101299]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:37.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:37 smithi032 ceph-mon[105748]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:37 smithi045 ceph-mon[95976]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:39.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:39 smithi032 ceph-mon[101299]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:39 smithi032 ceph-mon[105748]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:59:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:59:39] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.51.0" 2024-09-21T09:59:40.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:39 smithi045 ceph-mon[95976]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:41.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:41 smithi032 ceph-mon[101299]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:41 smithi032 ceph-mon[105748]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:41 smithi045 ceph-mon[95976]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:43.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:43 smithi032 ceph-mon[101299]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:43.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:43 smithi032 ceph-mon[105748]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:44.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:43 smithi045 ceph-mon[95976]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:45 smithi032 ceph-mon[105748]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:45 smithi032 ceph-mon[101299]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:45 smithi045 ceph-mon[95976]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:47 smithi032 ceph-mon[105748]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:47 smithi032 ceph-mon[101299]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:47 smithi045 ceph-mon[95976]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:48.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:48.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:49 smithi032 ceph-mon[101299]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:49 smithi032 ceph-mon[105748]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:59:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:59:49] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.51.0" 2024-09-21T09:59:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:49 smithi045 ceph-mon[95976]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:51 smithi032 ceph-mon[105748]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:51 smithi032 ceph-mon[101299]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:51 smithi045 ceph-mon[95976]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:53 smithi032 ceph-mon[105748]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:53 smithi032 ceph-mon[101299]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:53 smithi045 ceph-mon[95976]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:55.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:55 smithi032 ceph-mon[101299]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:55 smithi032 ceph-mon[105748]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:55 smithi045 ceph-mon[95976]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:57.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:57 smithi032 ceph-mon[101299]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:57.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:57 smithi032 ceph-mon[105748]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:57 smithi045 ceph-mon[95976]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:59.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 09:59:59 smithi032 ceph-mon[101299]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:59.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 09:59:59 smithi032 ceph-mon[105748]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:59:59.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 09:59:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:09:59:59] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.51.0" 2024-09-21T10:00:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 09:59:59 smithi045 ceph-mon[95976]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:00.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:00 smithi032 ceph-mon[101299]: overall HEALTH_OK 2024-09-21T10:00:00.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:00 smithi032 ceph-mon[105748]: overall HEALTH_OK 2024-09-21T10:00:01.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:00 smithi045 ceph-mon[95976]: overall HEALTH_OK 2024-09-21T10:00:01.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:01 smithi032 ceph-mon[101299]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:01.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:01 smithi032 ceph-mon[105748]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:01 smithi045 ceph-mon[95976]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:03.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:03 smithi032 ceph-mon[101299]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:03 smithi032 ceph-mon[105748]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:04.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:03 smithi045 ceph-mon[95976]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:04.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:04.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:04.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:05 smithi032 ceph-mon[105748]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:05 smithi032 ceph-mon[101299]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:05 smithi045 ceph-mon[95976]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:07.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:07 smithi032 ceph-mon[101299]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:07 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:07 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:07 smithi032 ceph-mon[105748]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:07 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:07 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:07 smithi045 ceph-mon[95976]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:07 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:07 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[101299]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[105748]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:09.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:09.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:09 smithi045 ceph-mon[95976]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:00:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:00:09] "GET /metrics HTTP/1.1" 200 33804 "" "Prometheus/2.51.0" 2024-09-21T10:00:11.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:11 smithi032 ceph-mon[101299]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:11.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:11 smithi032 ceph-mon[105748]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:11 smithi045 ceph-mon[95976]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:13 smithi032 ceph-mon[105748]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:13.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:13 smithi032 ceph-mon[101299]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:13 smithi045 ceph-mon[95976]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:15.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:15 smithi032 ceph-mon[105748]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:15.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:15 smithi032 ceph-mon[101299]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:15 smithi045 ceph-mon[95976]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:17.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:17 smithi032 ceph-mon[105748]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:17 smithi032 ceph-mon[101299]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:17.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:17 smithi045 ceph-mon[95976]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:19.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:19 smithi032 ceph-mon[101299]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:19.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:19.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:19 smithi032 ceph-mon[105748]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:19.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:19 smithi045 ceph-mon[95976]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:19.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:00:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:00:19] "GET /metrics HTTP/1.1" 200 33804 "" "Prometheus/2.51.0" 2024-09-21T10:00:21.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:21 smithi032 ceph-mon[101299]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:21.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:21 smithi032 ceph-mon[105748]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:21 smithi045 ceph-mon[95976]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:23 smithi032 ceph-mon[105748]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:23 smithi032 ceph-mon[101299]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:23 smithi045 ceph-mon[95976]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:25.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:25 smithi032 ceph-mon[101299]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:25 smithi032 ceph-mon[105748]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:25.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:25 smithi045 ceph-mon[95976]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:27.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:27 smithi032 ceph-mon[101299]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:27.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:27 smithi032 ceph-mon[105748]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:27 smithi045 ceph-mon[95976]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:29.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:29 smithi032 ceph-mon[101299]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:29.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:29 smithi032 ceph-mon[105748]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:29 smithi045 ceph-mon[95976]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:29.760 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:00:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:00:29] "GET /metrics HTTP/1.1" 200 33804 "" "Prometheus/2.51.0" 2024-09-21T10:00:30.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for all mons to upgrade' 2024-09-21T10:00:30.374 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:31.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:31 smithi032 ceph-mon[101299]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:31.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:31 smithi032 ceph-mon[105748]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:31.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:31 smithi045 ceph-mon[95976]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:32.054 INFO:teuthology.orchestra.run.smithi032.stdout:wait for all mons to upgrade 2024-09-21T10:00:32.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon' 2024-09-21T10:00:32.921 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:33.293 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:33 smithi032 ceph-mon[101299]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:33.293 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:33 smithi032 ceph-mon[105748]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:33 smithi045 ceph-mon[95976]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:34.348 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:34.348 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:35 smithi032 ceph-mon[101299]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:35 smithi032 ceph-mon[101299]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:35.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:35 smithi032 ceph-mon[105748]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:35.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:35 smithi032 ceph-mon[105748]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:35 smithi045 ceph-mon[95976]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:35 smithi045 ceph-mon[95976]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:36.654 INFO:teuthology.orchestra.run.smithi032.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:00:37.279 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T10:00:37.489 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:37.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[101299]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:37.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[101299]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:00:37.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:37.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:37.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:37.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:37.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:37.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[101299]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:00:37.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[105748]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:37.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[105748]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:00:37.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:37.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:37.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:37.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:37.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:37.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:37 smithi032 ceph-mon[105748]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:00:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:37 smithi045 ceph-mon[95976]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:37 smithi045 ceph-mon[95976]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:00:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:38.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:38.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:38.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:38.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:37 smithi045 ceph-mon[95976]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:00:39.893 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:39 smithi032 ceph-mon[105748]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:39.893 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:39 smithi032 ceph-mon[101299]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:39.893 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:00:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:00:39] "GET /metrics HTTP/1.1" 200 33800 "" "Prometheus/2.51.0" 2024-09-21T10:00:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:39 smithi045 ceph-mon[95976]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T10:00:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T10:00:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all mgr 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all mon 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all crash 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all rgw 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all iscsi 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all nfs 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all mgr 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all mon 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all crash 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all rgw 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all iscsi 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all nfs 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all node-exporter 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all prometheus 2024-09-21T10:00:41.212 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all alertmanager 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all grafana 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all loki 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all promtail 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Finalizing container_image settings 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:00:41.213 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all node-exporter 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.214 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all prometheus 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all alertmanager 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all grafana 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all loki 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all promtail 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Finalizing container_image settings 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:00:41.215 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:00:41.216 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: Upgrade: Complete! 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:41.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[105748]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:00:41.218 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: Upgrade: Complete! 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:41.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:40 smithi032 ceph-mon[101299]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:41.269 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'sleep 60' 2024-09-21T10:00:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T10:00:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T10:00:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all mgr 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all mon 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all crash 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all rgw 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all iscsi 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all nfs 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all node-exporter 2024-09-21T10:00:41.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all prometheus 2024-09-21T10:00:41.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all alertmanager 2024-09-21T10:00:41.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all grafana 2024-09-21T10:00:41.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all loki 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all promtail 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Finalizing container_image settings 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:00:41.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.290 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: Upgrade: Complete! 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:41.291 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:40 smithi045 ceph-mon[95976]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:41.479 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:43.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:43 smithi032 ceph-mon[101299]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:43.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:43 smithi032 ceph-mon[105748]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:43 smithi045 ceph-mon[95976]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:44.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:44 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:44.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:44 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:44.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:44 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:00:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:45 smithi045 ceph-mon[95976]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:45.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:45 smithi032 ceph-mon[101299]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:45 smithi032 ceph-mon[105748]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:47.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:47 smithi045 ceph-mon[95976]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:47 smithi032 ceph-mon[101299]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:47 smithi032 ceph-mon[105748]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:48.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:48.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:48.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:49 smithi045 ceph-mon[95976]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:49.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:49 smithi032 ceph-mon[101299]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:49 smithi032 ceph-mon[105748]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:00:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:00:49] "GET /metrics HTTP/1.1" 200 33733 "" "Prometheus/2.51.0" 2024-09-21T10:00:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:51 smithi045 ceph-mon[95976]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:51.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:51 smithi032 ceph-mon[101299]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:51 smithi032 ceph-mon[105748]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:53.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:53 smithi045 ceph-mon[95976]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:53 smithi032 ceph-mon[101299]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:53 smithi032 ceph-mon[105748]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:55.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:55 smithi045 ceph-mon[95976]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:55 smithi032 ceph-mon[101299]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:55 smithi032 ceph-mon[105748]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:57 smithi045 ceph-mon[95976]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:57.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:57 smithi032 ceph-mon[105748]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:57 smithi032 ceph-mon[101299]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:59.759 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:00:59 smithi032 ceph-mon[105748]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:59.759 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:00:59 smithi032 ceph-mon[101299]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:00:59.759 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:00:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:00:59] "GET /metrics HTTP/1.1" 200 33733 "" "Prometheus/2.51.0" 2024-09-21T10:00:59.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:00:59 smithi045 ceph-mon[95976]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:01 smithi045 ceph-mon[95976]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:01 smithi032 ceph-mon[105748]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:01 smithi032 ceph-mon[101299]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:03.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:03 smithi045 ceph-mon[95976]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:03 smithi032 ceph-mon[101299]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:03 smithi032 ceph-mon[105748]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:05 smithi045 ceph-mon[95976]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:05.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:05 smithi032 ceph-mon[101299]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:05 smithi032 ceph-mon[105748]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:07 smithi045 ceph-mon[95976]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:07 smithi032 ceph-mon[101299]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:07 smithi032 ceph-mon[105748]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:09 smithi045 ceph-mon[95976]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:09 smithi032 ceph-mon[101299]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:01:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:01:09] "GET /metrics HTTP/1.1" 200 33802 "" "Prometheus/2.51.0" 2024-09-21T10:01:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:09 smithi032 ceph-mon[105748]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:11.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:11 smithi045 ceph-mon[95976]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:11.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:11 smithi032 ceph-mon[101299]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:11.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:11 smithi032 ceph-mon[105748]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:13.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:13 smithi045 ceph-mon[95976]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:13.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:13 smithi032 ceph-mon[101299]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:13 smithi032 ceph-mon[105748]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:15.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:15 smithi045 ceph-mon[95976]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:15.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:15 smithi032 ceph-mon[101299]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:15.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:15 smithi032 ceph-mon[105748]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:17.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:17 smithi045 ceph-mon[95976]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:17.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:17 smithi032 ceph-mon[101299]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:17 smithi032 ceph-mon[105748]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:18.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:18.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:19 smithi032 ceph-mon[101299]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:19 smithi032 ceph-mon[105748]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:19.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:01:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:01:19] "GET /metrics HTTP/1.1" 200 33802 "" "Prometheus/2.51.0" 2024-09-21T10:01:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:19 smithi045 ceph-mon[95976]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:21 smithi032 ceph-mon[101299]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:21 smithi032 ceph-mon[105748]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:21 smithi045 ceph-mon[95976]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:23 smithi032 ceph-mon[101299]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:23.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:23 smithi032 ceph-mon[105748]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:23 smithi045 ceph-mon[95976]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:25.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:25 smithi032 ceph-mon[101299]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:25.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:25 smithi032 ceph-mon[105748]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:26.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:25 smithi045 ceph-mon[95976]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:27 smithi032 ceph-mon[101299]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:27 smithi032 ceph-mon[105748]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:27 smithi045 ceph-mon[95976]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:29.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:29 smithi032 ceph-mon[101299]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:29.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:29 smithi032 ceph-mon[105748]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:29.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:01:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:01:29] "GET /metrics HTTP/1.1" 200 33802 "" "Prometheus/2.51.0" 2024-09-21T10:01:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:29 smithi045 ceph-mon[95976]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:31.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:31 smithi032 ceph-mon[101299]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:31 smithi032 ceph-mon[105748]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:32.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:31 smithi045 ceph-mon[95976]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:33 smithi032 ceph-mon[105748]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:33 smithi032 ceph-mon[101299]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:33 smithi045 ceph-mon[95976]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:35.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:35 smithi032 ceph-mon[105748]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:35 smithi032 ceph-mon[101299]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:35 smithi045 ceph-mon[95976]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:37.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:37 smithi032 ceph-mon[101299]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:37.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:37 smithi032 ceph-mon[105748]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:37 smithi045 ceph-mon[95976]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:39 smithi032 ceph-mon[101299]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:39 smithi032 ceph-mon[105748]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:01:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:01:39] "GET /metrics HTTP/1.1" 200 33806 "" "Prometheus/2.51.0" 2024-09-21T10:01:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:39 smithi045 ceph-mon[95976]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:40.846 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:40.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:40.887 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:41 smithi045 ceph-mon[95976]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:41.909 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:41 smithi032 ceph-mon[101299]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:41.910 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:41 smithi032 ceph-mon[105748]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:43.719 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-21T10:01:43.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:43 smithi032 ceph-mon[101299]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:43.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:43 smithi032 ceph-mon[105748]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:44.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:43 smithi045 ceph-mon[95976]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:44.604 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:45.677 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:45 smithi045 ceph-mon[95976]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:45.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:45.679 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:45.679 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[101299]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:45.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[105748]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:45.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:45.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:45.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:45.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:46.374 INFO:teuthology.orchestra.run.smithi032.stdout:wait for half of osds to upgrade 2024-09-21T10:01:47.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4' 2024-09-21T10:01:47.221 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:47.649 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:47 smithi032 ceph-mon[101299]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:47.649 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:47 smithi032 ceph-mon[105748]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:47 smithi045 ceph-mon[95976]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:49.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:01:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:01:49] "GET /metrics HTTP/1.1" 200 33806 "" "Prometheus/2.51.0" 2024-09-21T10:01:49.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:49 smithi032 ceph-mon[101299]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:49.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:49.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:49 smithi032 ceph-mon[105748]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:49.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:49 smithi045 ceph-mon[95976]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:50.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:50 smithi032 ceph-mon[101299]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:50.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:50 smithi032 ceph-mon[105748]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:50.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:50 smithi045 ceph-mon[95976]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:51.078 INFO:teuthology.orchestra.run.smithi032.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:01:51.665 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[101299]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:51.665 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:51.666 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:51.666 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:51.666 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:51.666 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:51.666 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[105748]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:51.666 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:51.666 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:51.667 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:51.667 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:51.667 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:51.687 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T10:01:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:51 smithi045 ceph-mon[95976]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:51.906 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:52.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:52 smithi032 ceph-mon[101299]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:01:52.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:52 smithi032 ceph-mon[101299]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:01:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:52 smithi032 ceph-mon[105748]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:01:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:52 smithi032 ceph-mon[105748]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:01:52.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:52 smithi045 ceph-mon[95976]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:01:52.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:52 smithi045 ceph-mon[95976]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T10:01:53.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:53 smithi032 ceph-mon[101299]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:53 smithi032 ceph-mon[105748]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:53.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:53 smithi045 ceph-mon[95976]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:54.231 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T10:01:54.830 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:01:54.830 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (6m) 10s ago 12m 19.3M - 0.27.0 11f11916f8cd d6fa4f2dece8 2024-09-21T10:01:54.830 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (6m) 10s ago 12m 79.9M - 10.4.8 5aad1d7cf004 0ad08e43d776 2024-09-21T10:01:54.833 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (11m) 10s ago 11m 20.0M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T10:01:54.833 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (11m) 10s ago 11m 19.2M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T10:01:54.833 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (7m) 10s ago 16m 458M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T10:01:54.833 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (8m) 10s ago 18m 517M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T10:01:54.833 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (5m) 10s ago 18m 50.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 525e81f3c17e 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (3m) 10s ago 16m 42.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7115dfb7957a 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (5m) 10s ago 16m 44.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 233d45e2730d 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (7m) 10s ago 12m 10.1M - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (7m) 10s ago 12m 11.2M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (16m) 10s ago 16m 59.3M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 7c7ff33a9dde 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (15m) 10s ago 15m 54.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (15m) 10s ago 15m 58.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (14m) 10s ago 14m 46.9M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (14m) 10s ago 14m 55.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (14m) 10s ago 14m 49.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (13m) 10s ago 13m 49.1M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (13m) 10s ago 13m 52.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T10:01:54.834 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (7m) 10s ago 12m 45.6M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T10:01:55.195 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:01:55.196 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T10:01:55.197 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 10, 2024-09-21T10:01:55.197 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T10:01:55.197 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:01:55.197 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:01:55.505 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:01:55.505 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T10:01:55.505 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T10:01:55.505 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (4 remaining).", 2024-09-21T10:01:55.505 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T10:01:55.505 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "0/8 daemons upgraded", 2024-09-21T10:01:55.506 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:01:55.506 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T10:01:55.506 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:01:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T10:01:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T10:01:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all mgr 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all mon 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all crash 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: Upgrade: osd.0 is safe to restart 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3819181798' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.096 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all mgr 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all mon 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all crash 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: Upgrade: osd.0 is safe to restart 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3819181798' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T10:01:56.098 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all mgr 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all mon 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all crash 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: Upgrade: osd.0 is safe to restart 2024-09-21T10:01:56.099 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:56.100 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3819181798' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:56 smithi045 ceph-mon[95976]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:56 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:56 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T10:01:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:56 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:57.136 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:56 smithi032 ceph-mon[105748]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:57.136 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:56 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:57.136 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:56 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T10:01:57.137 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:56 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:57.137 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:56 smithi032 ceph-mon[101299]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:57.137 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:56 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:01:57.137 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:56 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T10:01:57.137 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:56 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:58.030 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:57 smithi032 ceph-mon[105748]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:58.030 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:57 smithi032 ceph-mon[105748]: Upgrade: Updating osd.0 2024-09-21T10:01:58.030 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:57 smithi032 ceph-mon[105748]: Deploying daemon osd.0 on smithi032 2024-09-21T10:01:58.031 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:57 smithi032 ceph-mon[101299]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:58.031 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:57 smithi032 ceph-mon[101299]: Upgrade: Updating osd.0 2024-09-21T10:01:58.031 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:57 smithi032 ceph-mon[101299]: Deploying daemon osd.0 on smithi032 2024-09-21T10:01:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:57 smithi045 ceph-mon[95976]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:01:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:57 smithi045 ceph-mon[95976]: Upgrade: Updating osd.0 2024-09-21T10:01:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:57 smithi045 ceph-mon[95976]: Deploying daemon osd.0 on smithi032 2024-09-21T10:01:59.286 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:01:59 smithi032 systemd[1]: Stopping Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:01:59.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:01:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:01:59] "GET /metrics HTTP/1.1" 200 33806 "" "Prometheus/2.51.0" 2024-09-21T10:01:59.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:01:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[43968]: 2024-09-21T10:01:59.283+0000 7f479ee43640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:01:59.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:01:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[43968]: 2024-09-21T10:01:59.283+0000 7f479ee43640 -1 osd.0 90 *** Got signal Terminated *** 2024-09-21T10:01:59.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:01:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[43968]: 2024-09-21T10:01:59.283+0000 7f479ee43640 -1 osd.0 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:02:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:59 smithi045 ceph-mon[95976]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:01:59 smithi045 ceph-mon[95976]: osd.0 marked itself down and dead 2024-09-21T10:02:00.196 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:59 smithi032 ceph-mon[101299]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:00.196 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:01:59 smithi032 ceph-mon[101299]: osd.0 marked itself down and dead 2024-09-21T10:02:00.197 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:59 smithi032 ceph-mon[105748]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:00.197 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:01:59 smithi032 ceph-mon[105748]: osd.0 marked itself down and dead 2024-09-21T10:02:00.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:00 smithi032 podman[115113]: 2024-09-21 10:02:00.193663387 +0000 UTC m=+1.015350571 container died 7c7ff33a9ddebf3db4f92f209b112ff508cda1922a6ed784e134e00778650dac (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8) 2024-09-21T10:02:00.754 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:00 smithi032 podman[115113]: 2024-09-21 10:02:00.583203546 +0000 UTC m=+1.404890727 container cleanup 7c7ff33a9ddebf3db4f92f209b112ff508cda1922a6ed784e134e00778650dac (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:02:00.755 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:00 smithi032 bash[115113]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:02:01.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:00 smithi045 ceph-mon[95976]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:02:01.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:00 smithi045 ceph-mon[95976]: osdmap e91: 8 total, 7 up, 8 in 2024-09-21T10:02:01.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:00 smithi032 ceph-mon[101299]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:02:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:00 smithi032 ceph-mon[101299]: osdmap e91: 8 total, 7 up, 8 in 2024-09-21T10:02:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:00 smithi032 ceph-mon[105748]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:02:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:00 smithi032 ceph-mon[105748]: osdmap e91: 8 total, 7 up, 8 in 2024-09-21T10:02:01.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:00 smithi032 podman[115132]: 2024-09-21 10:02:00.851533022 +0000 UTC m=+0.650009546 container remove 7c7ff33a9ddebf3db4f92f209b112ff508cda1922a6ed784e134e00778650dac (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8) 2024-09-21T10:02:01.473 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:01 smithi032 podman[115219]: 2024-09-21 10:02:01.108763722 +0000 UTC m=+0.016530086 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:02:01.474 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:01 smithi032 podman[115219]: 2024-09-21 10:02:01.210592731 +0000 UTC m=+0.118359086 container create 1ac4a04f40c2881b5ad45d552df65e4b61402729d9145725bdc3e2bf0b668e1e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T10:02:01.762 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:01 smithi032 podman[115219]: 2024-09-21 10:02:01.472400438 +0000 UTC m=+0.380166795 container init 1ac4a04f40c2881b5ad45d552df65e4b61402729d9145725bdc3e2bf0b668e1e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T10:02:01.763 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:01 smithi032 podman[115219]: 2024-09-21 10:02:01.476423139 +0000 UTC m=+0.384189498 container start 1ac4a04f40c2881b5ad45d552df65e4b61402729d9145725bdc3e2bf0b668e1e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T10:02:01.763 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:01 smithi032 podman[115219]: 2024-09-21 10:02:01.555572462 +0000 UTC m=+0.463338837 container attach 1ac4a04f40c2881b5ad45d552df65e4b61402729d9145725bdc3e2bf0b668e1e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T10:02:01.763 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:01 smithi032 podman[115219]: 2024-09-21 10:02:01.674588604 +0000 UTC m=+0.582354959 container died 1ac4a04f40c2881b5ad45d552df65e4b61402729d9145725bdc3e2bf0b668e1e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:02:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:01 smithi045 ceph-mon[95976]: pgmap v175: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:01 smithi045 ceph-mon[95976]: osdmap e92: 8 total, 7 up, 8 in 2024-09-21T10:02:02.035 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:01 smithi032 ceph-mon[105748]: pgmap v175: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:02.035 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:01 smithi032 ceph-mon[105748]: osdmap e92: 8 total, 7 up, 8 in 2024-09-21T10:02:02.036 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:01 smithi032 ceph-mon[101299]: pgmap v175: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:02.036 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:01 smithi032 ceph-mon[101299]: osdmap e92: 8 total, 7 up, 8 in 2024-09-21T10:02:02.627 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:02 smithi032 podman[115243]: 2024-09-21 10:02:02.309938673 +0000 UTC m=+0.625803882 container remove 1ac4a04f40c2881b5ad45d552df65e4b61402729d9145725bdc3e2bf0b668e1e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:02:02.627 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:02 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T10:02:02.627 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:02 smithi032 systemd[1]: Stopped Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:02:02.627 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:02 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0.service: Consumed 7.538s CPU time. 2024-09-21T10:02:02.627 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:02 smithi032 systemd[1]: Starting Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:02:02.930 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:02 smithi032 podman[115355]: 2024-09-21 10:02:02.653647367 +0000 UTC m=+0.016671211 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:02:02.930 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:02 smithi032 podman[115355]: 2024-09-21 10:02:02.786177521 +0000 UTC m=+0.149201355 container create 59bb41a4015a707e11a37772946d3d9470f41856a085edf76d694c09be3a6170 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T10:02:03.186 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 podman[115355]: 2024-09-21 10:02:03.030928121 +0000 UTC m=+0.393951963 container init 59bb41a4015a707e11a37772946d3d9470f41856a085edf76d694c09be3a6170 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:02:03.186 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 podman[115355]: 2024-09-21 10:02:03.034410551 +0000 UTC m=+0.397434386 container start 59bb41a4015a707e11a37772946d3d9470f41856a085edf76d694c09be3a6170 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T10:02:03.186 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 podman[115355]: 2024-09-21 10:02:03.097474616 +0000 UTC m=+0.460498463 container attach 59bb41a4015a707e11a37772946d3d9470f41856a085edf76d694c09be3a6170 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:02:03.186 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[115370]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:02:03.186 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 bash[115355]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:02:03.186 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[115370]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:02:03.187 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 bash[115355]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:02:03.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-mon[101299]: pgmap v177: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:03.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:03.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:03.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-mon[105748]: pgmap v177: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:03.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:03.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:03.932 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[115370]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:02:03.932 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 bash[115355]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:02:03.932 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[115370]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:02:03.932 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 bash[115355]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:02:03.932 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[115370]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:02:03.933 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 bash[115355]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:02:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:03 smithi045 ceph-mon[95976]: pgmap v177: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:04.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[115370]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:02:04.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 bash[115355]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:02:04.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[115370]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:02:04.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 bash[115355]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:02:04.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[115370]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:02:04.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 bash[115355]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:02:04.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[115370]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T10:02:04.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 bash[115355]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T10:02:04.208 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:03 smithi032 podman[115355]: 2024-09-21 10:02:03.959328164 +0000 UTC m=+1.322351998 container died 59bb41a4015a707e11a37772946d3d9470f41856a085edf76d694c09be3a6170 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T10:02:04.846 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:04 smithi032 ceph-mon[105748]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:04.846 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:04 smithi032 ceph-mon[101299]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:04.847 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:04 smithi032 podman[115496]: 2024-09-21 10:02:04.585542113 +0000 UTC m=+0.620245401 container remove 59bb41a4015a707e11a37772946d3d9470f41856a085edf76d694c09be3a6170 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:02:04.847 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:04 smithi032 podman[115558]: 2024-09-21 10:02:04.704879108 +0000 UTC m=+0.016787325 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:02:04.847 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:04 smithi032 podman[115558]: 2024-09-21 10:02:04.845001854 +0000 UTC m=+0.156910100 container create 0e29e15aee7335c2b9bd21eae3f5e8bce7549f6cec4437eef84ee18f06e2e718 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T10:02:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:04 smithi045 ceph-mon[95976]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:05.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:05 smithi032 podman[115558]: 2024-09-21 10:02:05.106449669 +0000 UTC m=+0.418357875 container init 0e29e15aee7335c2b9bd21eae3f5e8bce7549f6cec4437eef84ee18f06e2e718 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T10:02:05.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:05 smithi032 podman[115558]: 2024-09-21 10:02:05.109797436 +0000 UTC m=+0.421705642 container start 0e29e15aee7335c2b9bd21eae3f5e8bce7549f6cec4437eef84ee18f06e2e718 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T10:02:05.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-osd[115577]: -- 172.21.15.32:0/462080302 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55afadf01a40 con 0x55afaecd4400 2024-09-21T10:02:05.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:05 smithi032 bash[115558]: 0e29e15aee7335c2b9bd21eae3f5e8bce7549f6cec4437eef84ee18f06e2e718 2024-09-21T10:02:05.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:05 smithi032 systemd[1]: Started Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:02:06.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[105748]: pgmap v178: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[101299]: pgmap v178: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:05 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:05 smithi045 ceph-mon[95976]: pgmap v178: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:05 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:05 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:05 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:05 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:07.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:06 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[115573]: 2024-09-21T10:02:06.982+0000 7f9d9b051640 -1 Falling back to public interface 2024-09-21T10:02:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:07 smithi032 ceph-mon[101299]: pgmap v179: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:07 smithi032 ceph-mon[105748]: pgmap v179: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:07 smithi045 ceph-mon[95976]: pgmap v179: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:09.821 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:02:09] "GET /metrics HTTP/1.1" 200 33865 "" "Prometheus/2.51.0" 2024-09-21T10:02:10.136 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[101299]: pgmap v180: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:10.136 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.136 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.136 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.136 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.136 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[105748]: pgmap v180: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:10.137 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.137 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.137 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.137 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:09 smithi045 ceph-mon[95976]: pgmap v180: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:10.822 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[115573]: 2024-09-21T10:02:10.521+0000 7f9d9b051640 -1 osd.0 90 log_to_monitors true 2024-09-21T10:02:11.083 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:10 smithi032 ceph-mon[101299]: from='osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:02:11.083 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:10 smithi032 ceph-mon[105748]: from='osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:02:11.083 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:02:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[115573]: 2024-09-21T10:02:10.844+0000 7f9d92dfb640 -1 osd.0 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:02:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:10 smithi045 ceph-mon[95976]: from='osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:02:12.093 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:11 smithi032 ceph-mon[101299]: pgmap v181: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:12.093 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:11 smithi032 ceph-mon[101299]: from='osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:02:12.093 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:11 smithi032 ceph-mon[101299]: osdmap e93: 8 total, 7 up, 8 in 2024-09-21T10:02:12.093 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:11 smithi032 ceph-mon[101299]: from='osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:02:12.093 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:11 smithi032 ceph-mon[105748]: pgmap v181: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:12.093 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:11 smithi032 ceph-mon[105748]: from='osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:02:12.093 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:11 smithi032 ceph-mon[105748]: osdmap e93: 8 total, 7 up, 8 in 2024-09-21T10:02:12.093 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:11 smithi032 ceph-mon[105748]: from='osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:02:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:11 smithi045 ceph-mon[95976]: pgmap v181: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:11 smithi045 ceph-mon[95976]: from='osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:02:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:11 smithi045 ceph-mon[95976]: osdmap e93: 8 total, 7 up, 8 in 2024-09-21T10:02:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:11 smithi045 ceph-mon[95976]: from='osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:02:12.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T10:02:12.885+0000 7fce75a2e640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:02:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:02:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103] boot 2024-09-21T10:02:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T10:02:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:02:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[105748]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103] boot 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:12.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:12.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:12 smithi032 ceph-mon[101299]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T10:02:13.186 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:02:13.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: osd.0 [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103] boot 2024-09-21T10:02:13.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T10:02:13.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:02:13.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:13.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:13.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:13.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:13.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:13.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:12 smithi045 ceph-mon[95976]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[101299]: pgmap v184: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[105748]: pgmap v184: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:14.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:02:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:13 smithi045 ceph-mon[95976]: pgmap v184: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-21T10:02:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:02:15.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:14 smithi032 ceph-mon[105748]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:02:15.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:14 smithi032 ceph-mon[105748]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:02:15.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:14 smithi032 ceph-mon[101299]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:02:15.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:14 smithi032 ceph-mon[101299]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:02:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:14 smithi045 ceph-mon[95976]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:02:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:14 smithi045 ceph-mon[95976]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:02:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:15 smithi032 ceph-mon[101299]: pgmap v186: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-21T10:02:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:15 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:15 smithi032 ceph-mon[105748]: pgmap v186: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-21T10:02:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:15 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:15 smithi045 ceph-mon[95976]: pgmap v186: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-21T10:02:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:15 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:17.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:16 smithi032 ceph-mon[101299]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:17.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:16 smithi032 ceph-mon[101299]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-09-21T10:02:17.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:16 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:02:17.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:16 smithi032 ceph-mon[105748]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:17.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:16 smithi032 ceph-mon[105748]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-09-21T10:02:17.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:16 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:02:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:16 smithi045 ceph-mon[95976]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:16 smithi045 ceph-mon[95976]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-09-21T10:02:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:16 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:02:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:19 smithi045 ceph-mon[95976]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:19 smithi032 ceph-mon[101299]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:19 smithi032 ceph-mon[105748]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:19.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:02:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:02:19] "GET /metrics HTTP/1.1" 200 33864 "" "Prometheus/2.51.0" 2024-09-21T10:02:21.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:21 smithi045 ceph-mon[95976]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:21.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:21 smithi032 ceph-mon[105748]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:21 smithi032 ceph-mon[101299]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:23.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:23 smithi045 ceph-mon[95976]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:23.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:23 smithi032 ceph-mon[105748]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:23 smithi032 ceph-mon[101299]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:24 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:24.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:24 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:24 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:25 smithi045 ceph-mon[95976]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:25.841 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:25 smithi032 ceph-mon[105748]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:25.841 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:25 smithi032 ceph-mon[101299]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:25.842 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T10:02:26.432 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (7m) 17s ago 13m 19.3M - 0.27.0 11f11916f8cd d6fa4f2dece8 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (7m) 42s ago 12m 79.9M - 10.4.8 5aad1d7cf004 0ad08e43d776 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (11m) 17s ago 11m 20.2M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (11m) 42s ago 11m 19.2M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (8m) 42s ago 16m 458M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (8m) 17s ago 18m 518M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (6m) 17s ago 18m 53.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 525e81f3c17e 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (3m) 42s ago 17m 42.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7115dfb7957a 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (5m) 17s ago 17m 41.6M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 233d45e2730d 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (8m) 17s ago 12m 10.0M - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (8m) 42s ago 12m 11.2M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (21s) 17s ago 16m 11.3M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0e29e15aee73 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (16m) 17s ago 16m 54.3M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8df31b8f3dcb 2024-09-21T10:02:26.433 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (15m) 17s ago 15m 59.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T10:02:26.434 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (15m) 17s ago 15m 46.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T10:02:26.434 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (15m) 42s ago 15m 55.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T10:02:26.434 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (14m) 42s ago 14m 49.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T10:02:26.434 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (14m) 42s ago 14m 49.1M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T10:02:26.434 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (13m) 42s ago 13m 52.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T10:02:26.434 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (7m) 42s ago 13m 45.6M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T10:02:26.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:26 smithi032 ceph-mon[105748]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:26.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:26 smithi032 ceph-mon[101299]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:26.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:26 smithi045 ceph-mon[95976]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 7, 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T10:02:26.789 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T10:02:26.790 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:02:26.790 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T10:02:26.790 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 9, 2024-09-21T10:02:26.790 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 6 2024-09-21T10:02:26.790 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:02:26.790 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:02:27.088 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:02:27.088 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T10:02:27.088 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T10:02:27.089 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-09-21T10:02:27.089 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T10:02:27.089 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "1/8 daemons upgraded", 2024-09-21T10:02:27.089 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:02:27.089 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T10:02:27.089 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:02:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:27 smithi045 ceph-mon[95976]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:27 smithi045 ceph-mon[95976]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:27 smithi045 ceph-mon[95976]: from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1997277705' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:27.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:27 smithi032 ceph-mon[105748]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:27 smithi032 ceph-mon[105748]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:27 smithi032 ceph-mon[105748]: from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1997277705' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:27 smithi032 ceph-mon[101299]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:27 smithi032 ceph-mon[101299]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:27 smithi032 ceph-mon[101299]: from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1997277705' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:28.579 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:28 smithi032 ceph-mon[105748]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:28.579 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:28 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:02:28.579 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:28 smithi032 ceph-mon[101299]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:28.579 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:28 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:02:28.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:28 smithi045 ceph-mon[95976]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:28.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:28 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:02:29.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:02:29] "GET /metrics HTTP/1.1" 200 33864 "" "Prometheus/2.51.0" 2024-09-21T10:02:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[101299]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:02:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[101299]: Upgrade: osd.1 is safe to restart 2024-09-21T10:02:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[101299]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T10:02:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:29.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[105748]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:02:29.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[105748]: Upgrade: osd.1 is safe to restart 2024-09-21T10:02:29.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[105748]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:29.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:29.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T10:02:29.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:29 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:29 smithi045 ceph-mon[95976]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:02:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:29 smithi045 ceph-mon[95976]: Upgrade: osd.1 is safe to restart 2024-09-21T10:02:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:29 smithi045 ceph-mon[95976]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:29 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:29 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T10:02:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:29 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:30 smithi045 ceph-mon[95976]: Upgrade: Updating osd.1 2024-09-21T10:02:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:30 smithi045 ceph-mon[95976]: Deploying daemon osd.1 on smithi032 2024-09-21T10:02:30.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:30 smithi032 ceph-mon[101299]: Upgrade: Updating osd.1 2024-09-21T10:02:30.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:30 smithi032 ceph-mon[101299]: Deploying daemon osd.1 on smithi032 2024-09-21T10:02:30.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:30 smithi032 ceph-mon[105748]: Upgrade: Updating osd.1 2024-09-21T10:02:30.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:30 smithi032 ceph-mon[105748]: Deploying daemon osd.1 on smithi032 2024-09-21T10:02:31.740 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:31 smithi032 ceph-mon[105748]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:31.741 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:31 smithi032 ceph-mon[101299]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:31.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:31 smithi045 ceph-mon[95976]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:32.707 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:32 smithi032 systemd[1]: Stopping Ceph osd.1 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:02:32.707 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:32 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[49824]: 2024-09-21T10:02:32.686+0000 7f40a7de1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:02:32.707 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:32 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[49824]: 2024-09-21T10:02:32.686+0000 7f40a7de1640 -1 osd.1 95 *** Got signal Terminated *** 2024-09-21T10:02:32.707 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:32 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[49824]: 2024-09-21T10:02:32.686+0000 7f40a7de1640 -1 osd.1 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:02:33.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:33 smithi045 ceph-mon[95976]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:33.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:33 smithi045 ceph-mon[95976]: osd.1 marked itself down and dead 2024-09-21T10:02:33.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:33.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:33 smithi032 ceph-mon[101299]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:33.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:33 smithi032 ceph-mon[101299]: osd.1 marked itself down and dead 2024-09-21T10:02:33.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:33.908 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:33 smithi032 ceph-mon[105748]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:33.908 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:33 smithi032 ceph-mon[105748]: osd.1 marked itself down and dead 2024-09-21T10:02:33.908 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:34.189 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:33 smithi032 podman[119926]: 2024-09-21 10:02:33.90605184 +0000 UTC m=+1.315365107 container died 8df31b8f3dcb09da7a9456c21f5f73d7097f8392f87e707b3e7820e760b3f6a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T10:02:34.457 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:34 smithi032 podman[119926]: 2024-09-21 10:02:34.295248027 +0000 UTC m=+1.704561282 container cleanup 8df31b8f3dcb09da7a9456c21f5f73d7097f8392f87e707b3e7820e760b3f6a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1, org.label-schema.license=GPLv2, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:02:34.457 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:34 smithi032 bash[119926]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1 2024-09-21T10:02:34.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:34 smithi045 ceph-mon[95976]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:02:34.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:34 smithi045 ceph-mon[95976]: osdmap e96: 8 total, 7 up, 8 in 2024-09-21T10:02:34.923 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:34 smithi032 ceph-mon[101299]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:02:34.923 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:34 smithi032 ceph-mon[101299]: osdmap e96: 8 total, 7 up, 8 in 2024-09-21T10:02:34.923 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:34 smithi032 ceph-mon[105748]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:02:34.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:34 smithi032 ceph-mon[105748]: osdmap e96: 8 total, 7 up, 8 in 2024-09-21T10:02:34.924 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:34 smithi032 podman[119941]: 2024-09-21 10:02:34.580134644 +0000 UTC m=+0.663262492 container remove 8df31b8f3dcb09da7a9456c21f5f73d7097f8392f87e707b3e7820e760b3f6a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS) 2024-09-21T10:02:35.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:34 smithi032 podman[120029]: 2024-09-21 10:02:34.824846008 +0000 UTC m=+0.017003121 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:02:35.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:34 smithi032 podman[120029]: 2024-09-21 10:02:34.936926511 +0000 UTC m=+0.129083624 container create 16bdf0564780d6cdeb321f76352d603a2abd2fa06f070991225cd2ec1b8617c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T10:02:35.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:35 smithi032 podman[120029]: 2024-09-21 10:02:35.167116206 +0000 UTC m=+0.359273326 container init 16bdf0564780d6cdeb321f76352d603a2abd2fa06f070991225cd2ec1b8617c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T10:02:35.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:35 smithi032 podman[120029]: 2024-09-21 10:02:35.170867716 +0000 UTC m=+0.363024834 container start 16bdf0564780d6cdeb321f76352d603a2abd2fa06f070991225cd2ec1b8617c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T10:02:35.522 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:35 smithi032 podman[120029]: 2024-09-21 10:02:35.259056778 +0000 UTC m=+0.451213903 container attach 16bdf0564780d6cdeb321f76352d603a2abd2fa06f070991225cd2ec1b8617c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=main-62da54b, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T10:02:35.522 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:35 smithi032 podman[120029]: 2024-09-21 10:02:35.358614086 +0000 UTC m=+0.550771201 container died 16bdf0564780d6cdeb321f76352d603a2abd2fa06f070991225cd2ec1b8617c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_CLEAN=True, ceph=True) 2024-09-21T10:02:35.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:35 smithi045 ceph-mon[95976]: pgmap v197: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:35.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:35 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T10:02:35.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:35 smithi045 ceph-mon[95976]: osdmap e97: 8 total, 7 up, 8 in 2024-09-21T10:02:35.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:35 smithi032 ceph-mon[105748]: pgmap v197: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:35.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:35 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T10:02:35.939 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:35 smithi032 ceph-mon[105748]: osdmap e97: 8 total, 7 up, 8 in 2024-09-21T10:02:35.940 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:35 smithi032 ceph-mon[101299]: pgmap v197: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:35.940 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:35 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T10:02:35.940 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:35 smithi032 ceph-mon[101299]: osdmap e97: 8 total, 7 up, 8 in 2024-09-21T10:02:36.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:35 smithi032 podman[120051]: 2024-09-21 10:02:35.938226534 +0000 UTC m=+0.573321865 container remove 16bdf0564780d6cdeb321f76352d603a2abd2fa06f070991225cd2ec1b8617c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:02:36.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:35 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T10:02:36.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:35 smithi032 systemd[1]: Stopped Ceph osd.1 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:02:36.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:35 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.1.service: Consumed 8.119s CPU time. 2024-09-21T10:02:36.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 systemd[1]: Starting Ceph osd.1 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:02:36.702 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:36 smithi032 ceph-mon[101299]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:36.702 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:36 smithi032 ceph-mon[105748]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:36.703 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 podman[120153]: 2024-09-21 10:02:36.28529336 +0000 UTC m=+0.016181274 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:02:36.703 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 podman[120153]: 2024-09-21 10:02:36.389540552 +0000 UTC m=+0.120428450 container create a453b49a3f582192da73a317813ddfefa02c98d9579102eb13a35322862ab33c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T10:02:36.703 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 podman[120153]: 2024-09-21 10:02:36.617293304 +0000 UTC m=+0.348181212 container init a453b49a3f582192da73a317813ddfefa02c98d9579102eb13a35322862ab33c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T10:02:36.703 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 podman[120153]: 2024-09-21 10:02:36.620871259 +0000 UTC m=+0.351759160 container start a453b49a3f582192da73a317813ddfefa02c98d9579102eb13a35322862ab33c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-21T10:02:36.703 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 podman[120153]: 2024-09-21 10:02:36.700565551 +0000 UTC m=+0.431453469 container attach a453b49a3f582192da73a317813ddfefa02c98d9579102eb13a35322862ab33c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=main-62da54b) 2024-09-21T10:02:36.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate[120168]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:02:36.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 bash[120153]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:02:36.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate[120168]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:02:36.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:36 smithi032 bash[120153]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:02:37.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:36 smithi045 ceph-mon[95976]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:37.522 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate[120168]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:02:37.523 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 bash[120153]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:02:37.523 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate[120168]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T10:02:37.523 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 bash[120153]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T10:02:37.523 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate[120168]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T10:02:37.523 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 bash[120153]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T10:02:37.847 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:37 smithi032 ceph-mon[101299]: pgmap v199: 73 pgs: 2 active+undersized, 28 peering, 2 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:02:37.847 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:37 smithi032 ceph-mon[105748]: pgmap v199: 73 pgs: 2 active+undersized, 28 peering, 2 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:02:37.848 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate[120168]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:02:37.848 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 bash[120153]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:02:37.848 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate[120168]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T10:02:37.848 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 bash[120153]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T10:02:37.848 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate[120168]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:02:37.848 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 bash[120153]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:02:37.848 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate[120168]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T10:02:37.848 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 bash[120153]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T10:02:37.848 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:37 smithi032 podman[120153]: 2024-09-21 10:02:37.551465551 +0000 UTC m=+1.282353457 container died a453b49a3f582192da73a317813ddfefa02c98d9579102eb13a35322862ab33c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-21T10:02:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:37 smithi045 ceph-mon[95976]: pgmap v199: 73 pgs: 2 active+undersized, 28 peering, 2 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:02:38.457 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:38 smithi032 podman[120304]: 2024-09-21 10:02:38.175416593 +0000 UTC m=+0.612849015 container remove a453b49a3f582192da73a317813ddfefa02c98d9579102eb13a35322862ab33c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T10:02:38.457 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:38 smithi032 podman[120367]: 2024-09-21 10:02:38.292733272 +0000 UTC m=+0.016665670 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:02:38.457 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:38 smithi032 podman[120367]: 2024-09-21 10:02:38.414747811 +0000 UTC m=+0.138680189 container create 7166dcc673030830b63b5acc78407fde5b8a8acd94a055568d67d8103df85832 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:02:38.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:38 smithi032 podman[120367]: 2024-09-21 10:02:38.675684096 +0000 UTC m=+0.399616488 container init 7166dcc673030830b63b5acc78407fde5b8a8acd94a055568d67d8103df85832 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b) 2024-09-21T10:02:38.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:38 smithi032 podman[120367]: 2024-09-21 10:02:38.679112201 +0000 UTC m=+0.403044582 container start 7166dcc673030830b63b5acc78407fde5b8a8acd94a055568d67d8103df85832 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T10:02:38.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:38 smithi032 ceph-osd[120386]: -- 172.21.15.32:0/1516828894 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561a1fb09a40 con 0x561a208d8000 2024-09-21T10:02:38.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:38 smithi032 bash[120367]: 7166dcc673030830b63b5acc78407fde5b8a8acd94a055568d67d8103df85832 2024-09-21T10:02:38.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:38 smithi032 systemd[1]: Started Ceph osd.1 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:02:39.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:02:39] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.51.0" 2024-09-21T10:02:40.092 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[101299]: pgmap v200: 73 pgs: 2 active+undersized, 28 peering, 2 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:02:40.092 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:40.092 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:40.092 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:40.092 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:40.092 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[105748]: pgmap v200: 73 pgs: 2 active+undersized, 28 peering, 2 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:02:40.093 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:40.093 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:40.093 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:40.093 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:39 smithi045 ceph-mon[95976]: pgmap v200: 73 pgs: 2 active+undersized, 28 peering, 2 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:02:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:40.456 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:40 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[120382]: 2024-09-21T10:02:40.090+0000 7f453f85d640 -1 Falling back to public interface 2024-09-21T10:02:41.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:40 smithi032 ceph-mon[101299]: pgmap v201: 73 pgs: 26 active+undersized, 6 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-21T10:02:41.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:40 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2024-09-21T10:02:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:40 smithi032 ceph-mon[105748]: pgmap v201: 73 pgs: 26 active+undersized, 6 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-21T10:02:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:40 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2024-09-21T10:02:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:40 smithi045 ceph-mon[95976]: pgmap v201: 73 pgs: 26 active+undersized, 6 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-21T10:02:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:40 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2024-09-21T10:02:43.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[101299]: pgmap v202: 73 pgs: 26 active+undersized, 6 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-21T10:02:43.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.589 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[105748]: pgmap v202: 73 pgs: 26 active+undersized, 6 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-21T10:02:43.590 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.590 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.590 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.590 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:43 smithi045 ceph-mon[95976]: pgmap v202: 73 pgs: 26 active+undersized, 6 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-21T10:02:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:43 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:43 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:43 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:43 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:43.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:43 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[120382]: 2024-09-21T10:02:43.630+0000 7f453f85d640 -1 osd.1 95 log_to_monitors true 2024-09-21T10:02:44.651 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:44 smithi032 ceph-mon[105748]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T10:02:44.651 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:44 smithi032 ceph-mon[101299]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T10:02:44.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:44 smithi045 ceph-mon[95976]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T10:02:44.931 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:02:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[120382]: 2024-09-21T10:02:44.649+0000 7f45335ff640 -1 osd.1 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:02:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[105748]: pgmap v203: 73 pgs: 26 active+undersized, 6 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-21T10:02:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[105748]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:02:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[105748]: osdmap e98: 8 total, 7 up, 8 in 2024-09-21T10:02:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[105748]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:02:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[105748]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' 2024-09-21T10:02:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[101299]: pgmap v203: 73 pgs: 26 active+undersized, 6 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-21T10:02:45.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[101299]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:02:45.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[101299]: osdmap e98: 8 total, 7 up, 8 in 2024-09-21T10:02:45.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[101299]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:02:45.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:45 smithi032 ceph-mon[101299]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' 2024-09-21T10:02:45.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:45 smithi045 ceph-mon[95976]: pgmap v203: 73 pgs: 26 active+undersized, 6 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-21T10:02:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:45 smithi045 ceph-mon[95976]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:02:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:45 smithi045 ceph-mon[95976]: osdmap e98: 8 total, 7 up, 8 in 2024-09-21T10:02:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:45 smithi045 ceph-mon[95976]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:02:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:45 smithi045 ceph-mon[95976]: from='osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778]' entity='osd.1' 2024-09-21T10:02:46.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T10:02:46.259+0000 7fce75a2e640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778] boot 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778] boot 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:46.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:46.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:46.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:46.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:46.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:02:46.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:46 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:02:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: osd.1 [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778] boot 2024-09-21T10:02:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T10:02:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T10:02:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:46.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:46.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:46.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:46.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:46.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:46.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:02:46.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:46 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:47.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:47 smithi032 ceph-mon[101299]: pgmap v206: 73 pgs: 3 peering, 24 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:02:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:47 smithi032 ceph-mon[101299]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:02:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:47 smithi032 ceph-mon[101299]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:02:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:47 smithi032 ceph-mon[101299]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T10:02:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:47 smithi032 ceph-mon[105748]: pgmap v206: 73 pgs: 3 peering, 24 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:02:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:47 smithi032 ceph-mon[105748]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:02:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:47 smithi032 ceph-mon[105748]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:02:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:47 smithi032 ceph-mon[105748]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T10:02:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:47 smithi045 ceph-mon[95976]: pgmap v206: 73 pgs: 3 peering, 24 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:02:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:47 smithi045 ceph-mon[95976]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:02:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:47 smithi045 ceph-mon[95976]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:02:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:47 smithi045 ceph-mon[95976]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T10:02:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:49 smithi045 ceph-mon[95976]: pgmap v208: 73 pgs: 3 peering, 24 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:02:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:49 smithi032 ceph-mon[101299]: pgmap v208: 73 pgs: 3 peering, 24 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:02:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:02:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:02:49] "GET /metrics HTTP/1.1" 200 33778 "" "Prometheus/2.51.0" 2024-09-21T10:02:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:49 smithi032 ceph-mon[105748]: pgmap v208: 73 pgs: 3 peering, 24 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:02:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:50.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:50 smithi045 ceph-mon[95976]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded) 2024-09-21T10:02:50.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:50 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:02:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:50 smithi032 ceph-mon[101299]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded) 2024-09-21T10:02:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:50 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:02:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:50 smithi032 ceph-mon[105748]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded) 2024-09-21T10:02:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:50 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:02:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:51 smithi045 ceph-mon[95976]: pgmap v209: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:51.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:51 smithi032 ceph-mon[105748]: pgmap v209: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:51 smithi032 ceph-mon[101299]: pgmap v209: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:53.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:53 smithi045 ceph-mon[95976]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:53.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:53 smithi032 ceph-mon[105748]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:53 smithi032 ceph-mon[101299]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:55.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:55 smithi045 ceph-mon[95976]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:55 smithi032 ceph-mon[105748]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:55 smithi032 ceph-mon[101299]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:57.416 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T10:02:57.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:57 smithi032 ceph-mon[105748]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:57.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:57 smithi032 ceph-mon[101299]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:57 smithi045 ceph-mon[95976]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:58.003 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (7m) 15s ago 13m 19.3M - 0.27.0 11f11916f8cd d6fa4f2dece8 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (7m) 73s ago 13m 79.9M - 10.4.8 5aad1d7cf004 0ad08e43d776 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (12m) 15s ago 12m 20.4M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (12m) 73s ago 12m 19.2M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (9m) 73s ago 17m 458M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (9m) 15s ago 19m 518M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (6m) 15s ago 19m 55.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 525e81f3c17e 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (4m) 73s ago 17m 42.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7115dfb7957a 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (6m) 15s ago 17m 45.1M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 233d45e2730d 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (8m) 15s ago 13m 11.0M - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (8m) 73s ago 13m 11.2M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (53s) 15s ago 17m 40.8M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0e29e15aee73 2024-09-21T10:02:58.004 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (19s) 15s ago 16m 11.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7166dcc67303 2024-09-21T10:02:58.005 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (16m) 15s ago 16m 59.9M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 24312dae81e0 2024-09-21T10:02:58.005 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (16m) 15s ago 16m 48.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T10:02:58.005 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (15m) 73s ago 15m 55.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T10:02:58.005 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (15m) 73s ago 15m 49.5M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T10:02:58.005 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (14m) 73s ago 14m 49.1M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T10:02:58.005 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (14m) 73s ago 14m 52.0M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T10:02:58.005 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (8m) 73s ago 13m 45.6M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T10:02:58.359 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:02:58.359 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T10:02:58.359 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T10:02:58.359 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:02:58.359 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T10:02:58.359 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T10:02:58.359 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:02:58.359 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T10:02:58.359 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 6, 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8, 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 7 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:02:58.360 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:02:58.661 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:02:58.661 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T10:02:58.661 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T10:02:58.662 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-21T10:02:58.662 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T10:02:58.662 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "2/8 daemons upgraded", 2024-09-21T10:02:58.662 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:02:58.662 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T10:02:58.662 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:02:58.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:58 smithi032 ceph-mon[105748]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:58.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:58 smithi032 ceph-mon[105748]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:58.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:58 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4160294159' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:58.662 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:58 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:58.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:58 smithi032 ceph-mon[101299]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:58.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:58 smithi032 ceph-mon[101299]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:58.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:58 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4160294159' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:58.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:58 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:58.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:58 smithi045 ceph-mon[95976]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:58.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:58 smithi045 ceph-mon[95976]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:58.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:58 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4160294159' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:02:58.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:58 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:02:59.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:59 smithi045 ceph-mon[95976]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:59 smithi045 ceph-mon[95976]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:02:59 smithi045 ceph-mon[95976]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:59.858 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:02:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:02:59] "GET /metrics HTTP/1.1" 200 33778 "" "Prometheus/2.51.0" 2024-09-21T10:02:59.858 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:59 smithi032 ceph-mon[101299]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:59.858 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:59 smithi032 ceph-mon[101299]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:59.858 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:02:59 smithi032 ceph-mon[101299]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:59.858 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:59 smithi032 ceph-mon[105748]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:02:59.858 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:59 smithi032 ceph-mon[105748]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:02:59.858 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:02:59 smithi032 ceph-mon[105748]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:01 smithi045 ceph-mon[95976]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:01 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:03:01.871 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:01 smithi032 ceph-mon[105748]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:01.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:01 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:03:01.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:01 smithi032 ceph-mon[101299]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:01.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:01 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:03:02.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:02 smithi045 ceph-mon[95976]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:03:02.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:02 smithi045 ceph-mon[95976]: Upgrade: osd.2 is safe to restart 2024-09-21T10:03:02.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:02 smithi032 ceph-mon[105748]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:03:02.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:02 smithi032 ceph-mon[105748]: Upgrade: osd.2 is safe to restart 2024-09-21T10:03:02.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:02 smithi032 ceph-mon[101299]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:03:02.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:02 smithi032 ceph-mon[101299]: Upgrade: osd.2 is safe to restart 2024-09-21T10:03:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[101299]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[101299]: Upgrade: Updating osd.2 2024-09-21T10:03:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T10:03:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[101299]: Deploying daemon osd.2 on smithi032 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[105748]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[105748]: Upgrade: Updating osd.2 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[105748]: Deploying daemon osd.2 on smithi032 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:03.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:03 smithi045 ceph-mon[95976]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:03 smithi045 ceph-mon[95976]: Upgrade: Updating osd.2 2024-09-21T10:03:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T10:03:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:03 smithi045 ceph-mon[95976]: Deploying daemon osd.2 on smithi032 2024-09-21T10:03:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:05.707 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:05 smithi032 systemd[1]: Stopping Ceph osd.2 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:03:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:05 smithi045 ceph-mon[95976]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:06.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:05 smithi032 ceph-mon[101299]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:05 smithi032 ceph-mon[105748]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:06.207 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[55639]: 2024-09-21T10:03:05.803+0000 7f8dabcf3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:03:06.207 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[55639]: 2024-09-21T10:03:05.803+0000 7f8dabcf3640 -1 osd.2 100 *** Got signal Terminated *** 2024-09-21T10:03:06.207 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[55639]: 2024-09-21T10:03:05.803+0000 7f8dabcf3640 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:03:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:06 smithi045 ceph-mon[95976]: osd.2 marked itself down and dead 2024-09-21T10:03:07.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:06 smithi032 ceph-mon[105748]: osd.2 marked itself down and dead 2024-09-21T10:03:07.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:06 smithi032 ceph-mon[101299]: osd.2 marked itself down and dead 2024-09-21T10:03:07.524 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:07 smithi032 podman[124311]: 2024-09-21 10:03:07.208834414 +0000 UTC m=+1.531451159 container died 24312dae81e05f15784a91df443d2d906d71770daed83f8bd7f23be5bbd2c936 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:03:07.882 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:07 smithi032 ceph-mon[105748]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:07.882 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:07 smithi032 ceph-mon[105748]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:03:07.882 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:07 smithi032 ceph-mon[105748]: osdmap e101: 8 total, 7 up, 8 in 2024-09-21T10:03:07.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:07 smithi032 ceph-mon[101299]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:07.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:07 smithi032 ceph-mon[101299]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:03:07.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:07 smithi032 ceph-mon[101299]: osdmap e101: 8 total, 7 up, 8 in 2024-09-21T10:03:07.883 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:07 smithi032 podman[124311]: 2024-09-21 10:03:07.628853612 +0000 UTC m=+1.951470348 container cleanup 24312dae81e05f15784a91df443d2d906d71770daed83f8bd7f23be5bbd2c936 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T10:03:07.883 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:07 smithi032 bash[124311]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2 2024-09-21T10:03:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:07 smithi045 ceph-mon[95976]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:07 smithi045 ceph-mon[95976]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:03:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:07 smithi045 ceph-mon[95976]: osdmap e101: 8 total, 7 up, 8 in 2024-09-21T10:03:08.206 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:07 smithi032 podman[124326]: 2024-09-21 10:03:07.880603722 +0000 UTC m=+0.669854322 container remove 24312dae81e05f15784a91df443d2d906d71770daed83f8bd7f23be5bbd2c936 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T10:03:08.561 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:08 smithi032 podman[124415]: 2024-09-21 10:03:08.208439401 +0000 UTC m=+0.016320944 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:03:08.561 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:08 smithi032 podman[124415]: 2024-09-21 10:03:08.32347941 +0000 UTC m=+0.131360936 container create db7e9e85a2c702eb24aa8a9f0b0897584606d4c31d79bfc804cef1782fdac32b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T10:03:08.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:08 smithi032 ceph-mon[101299]: osdmap e102: 8 total, 7 up, 8 in 2024-09-21T10:03:08.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:08 smithi032 ceph-mon[105748]: osdmap e102: 8 total, 7 up, 8 in 2024-09-21T10:03:08.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:08 smithi032 podman[124415]: 2024-09-21 10:03:08.559518747 +0000 UTC m=+0.367400273 container init db7e9e85a2c702eb24aa8a9f0b0897584606d4c31d79bfc804cef1782fdac32b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T10:03:08.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:08 smithi032 podman[124415]: 2024-09-21 10:03:08.563773335 +0000 UTC m=+0.371654861 container start db7e9e85a2c702eb24aa8a9f0b0897584606d4c31d79bfc804cef1782fdac32b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b) 2024-09-21T10:03:08.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:08 smithi032 podman[124415]: 2024-09-21 10:03:08.651106754 +0000 UTC m=+0.458988298 container attach db7e9e85a2c702eb24aa8a9f0b0897584606d4c31d79bfc804cef1782fdac32b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:03:08.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:08 smithi032 podman[124415]: 2024-09-21 10:03:08.753441866 +0000 UTC m=+0.561323404 container died db7e9e85a2c702eb24aa8a9f0b0897584606d4c31d79bfc804cef1782fdac32b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_CLEAN=True) 2024-09-21T10:03:09.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:08 smithi045 ceph-mon[95976]: osdmap e102: 8 total, 7 up, 8 in 2024-09-21T10:03:09.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:03:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:03:09] "GET /metrics HTTP/1.1" 200 33926 "" "Prometheus/2.51.0" 2024-09-21T10:03:09.707 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:09 smithi032 podman[124438]: 2024-09-21 10:03:09.380451195 +0000 UTC m=+0.620275872 container remove db7e9e85a2c702eb24aa8a9f0b0897584606d4c31d79bfc804cef1782fdac32b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T10:03:09.707 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:09 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T10:03:09.708 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:09 smithi032 systemd[1]: Stopped Ceph osd.2 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:03:09.708 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:09 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.2.service: Consumed 7.137s CPU time. 2024-09-21T10:03:09.708 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:09 smithi032 systemd[1]: Starting Ceph osd.2 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:03:10.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:09 smithi045 ceph-mon[95976]: pgmap v220: 73 pgs: 2 stale+active+clean, 71 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:10.068 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:09 smithi032 ceph-mon[105748]: pgmap v220: 73 pgs: 2 stale+active+clean, 71 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:10.068 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:09 smithi032 ceph-mon[101299]: pgmap v220: 73 pgs: 2 stale+active+clean, 71 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:10.069 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:09 smithi032 podman[124537]: 2024-09-21 10:03:09.831498929 +0000 UTC m=+0.113607886 container create fef824f1895afc85061d71c93b4fbaca143f848d9dd3f89a4a16548c25351c7a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T10:03:10.069 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:09 smithi032 podman[124537]: 2024-09-21 10:03:09.734911901 +0000 UTC m=+0.017020830 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:03:10.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 podman[124537]: 2024-09-21 10:03:10.06724433 +0000 UTC m=+0.349353244 container init fef824f1895afc85061d71c93b4fbaca143f848d9dd3f89a4a16548c25351c7a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:03:10.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 podman[124537]: 2024-09-21 10:03:10.070715765 +0000 UTC m=+0.352824682 container start fef824f1895afc85061d71c93b4fbaca143f848d9dd3f89a4a16548c25351c7a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T10:03:10.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 podman[124537]: 2024-09-21 10:03:10.150608483 +0000 UTC m=+0.432717405 container attach fef824f1895afc85061d71c93b4fbaca143f848d9dd3f89a4a16548c25351c7a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-21T10:03:10.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate[124553]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:03:10.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 bash[124537]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:03:10.458 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate[124553]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:03:10.458 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 bash[124537]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:03:10.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-mon[101299]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:10.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-mon[105748]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:10.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate[124553]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T10:03:10.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 bash[124537]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T10:03:10.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate[124553]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T10:03:10.958 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 bash[124537]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T10:03:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:10 smithi045 ceph-mon[95976]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:11.323 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate[124553]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T10:03:11.323 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 bash[124537]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T10:03:11.323 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate[124553]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T10:03:11.323 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 bash[124537]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T10:03:11.324 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate[124553]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T10:03:11.324 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 bash[124537]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T10:03:11.324 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate[124553]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T10:03:11.324 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 bash[124537]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T10:03:11.324 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate[124553]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T10:03:11.324 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:10 smithi032 bash[124537]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T10:03:11.324 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:11 smithi032 podman[124537]: 2024-09-21 10:03:11.012648104 +0000 UTC m=+1.294757029 container died fef824f1895afc85061d71c93b4fbaca143f848d9dd3f89a4a16548c25351c7a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:03:11.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:11 smithi032 ceph-mon[101299]: pgmap v221: 73 pgs: 15 active+undersized, 2 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-21T10:03:11.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:11 smithi032 ceph-mon[105748]: pgmap v221: 73 pgs: 15 active+undersized, 2 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-21T10:03:11.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:11 smithi032 podman[124693]: 2024-09-21 10:03:11.613467291 +0000 UTC m=+0.594308500 container remove fef824f1895afc85061d71c93b4fbaca143f848d9dd3f89a4a16548c25351c7a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-activate, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-21T10:03:11.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:11 smithi032 podman[124753]: 2024-09-21 10:03:11.733384633 +0000 UTC m=+0.016966012 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:03:11.957 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:11 smithi032 podman[124753]: 2024-09-21 10:03:11.830978217 +0000 UTC m=+0.114559583 container create 84d2b69ff1bafc5ca35c22182b9e7244a9dcbd954f47da051c4250c68b1a4505 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T10:03:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:11 smithi045 ceph-mon[95976]: pgmap v221: 73 pgs: 15 active+undersized, 2 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-21T10:03:12.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:12 smithi032 podman[124753]: 2024-09-21 10:03:12.108484636 +0000 UTC m=+0.392066012 container init 84d2b69ff1bafc5ca35c22182b9e7244a9dcbd954f47da051c4250c68b1a4505 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T10:03:12.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:12 smithi032 podman[124753]: 2024-09-21 10:03:12.111887072 +0000 UTC m=+0.395468436 container start 84d2b69ff1bafc5ca35c22182b9e7244a9dcbd954f47da051c4250c68b1a4505 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-21T10:03:12.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:12 smithi032 ceph-osd[124772]: -- 172.21.15.32:0/311765700 <== mon.2 v2:172.21.15.32:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558de2b39a40 con 0x558de3908000 2024-09-21T10:03:12.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:12 smithi032 bash[124753]: 84d2b69ff1bafc5ca35c22182b9e7244a9dcbd954f47da051c4250c68b1a4505 2024-09-21T10:03:12.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:12 smithi032 systemd[1]: Started Ceph osd.2 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:03:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:13 smithi045 ceph-mon[95976]: pgmap v222: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:13.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[105748]: pgmap v222: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:13.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:13.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:13.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:13.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:13.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[101299]: pgmap v222: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:13.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:13.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:13.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:13.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:14.206 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[124768]: 2024-09-21T10:03:13.744+0000 7f89753a7640 -1 Falling back to public interface 2024-09-21T10:03:15.444 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:15 smithi032 ceph-mon[105748]: pgmap v223: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:15.444 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:15 smithi032 ceph-mon[101299]: pgmap v223: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:15 smithi045 ceph-mon[95976]: pgmap v223: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:16.955 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.955 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.955 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.955 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.955 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:16.955 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.955 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.955 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.955 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:16.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:16.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:16 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:16 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:16 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:16 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:16 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:16 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:16 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:16 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:17.708 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[124768]: 2024-09-21T10:03:17.294+0000 7f89753a7640 -1 osd.2 100 log_to_monitors true 2024-09-21T10:03:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:17 smithi045 ceph-mon[95976]: pgmap v224: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:17 smithi045 ceph-mon[95976]: from='osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T10:03:18.047 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:17 smithi032 ceph-mon[101299]: pgmap v224: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:18.047 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:17 smithi032 ceph-mon[101299]: from='osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T10:03:18.048 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:17 smithi032 ceph-mon[105748]: pgmap v224: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:18.048 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:17 smithi032 ceph-mon[105748]: from='osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T10:03:18.048 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:03:17 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[124768]: 2024-09-21T10:03:17.761+0000 7f896d151640 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:03:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:18 smithi045 ceph-mon[95976]: from='osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T10:03:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:18 smithi045 ceph-mon[95976]: osdmap e103: 8 total, 7 up, 8 in 2024-09-21T10:03:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:18 smithi045 ceph-mon[95976]: from='osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:03:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[101299]: from='osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T10:03:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[101299]: osdmap e103: 8 total, 7 up, 8 in 2024-09-21T10:03:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[101299]: from='osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:03:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:19.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[105748]: from='osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T10:03:19.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[105748]: osdmap e103: 8 total, 7 up, 8 in 2024-09-21T10:03:19.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[105748]: from='osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:03:19.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:19.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:19.663 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:03:19] "GET /metrics HTTP/1.1" 200 33778 "" "Prometheus/2.51.0" 2024-09-21T10:03:19.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T10:03:19.806+0000 7fce75a2e640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:03:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: pgmap v226: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:03:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490] boot 2024-09-21T10:03:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T10:03:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T10:03:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: pgmap v226: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490] boot 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:19.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:19.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:19.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: pgmap v226: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-21T10:03:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:03:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: osd.2 [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490] boot 2024-09-21T10:03:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:20 smithi045 ceph-mon[95976]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T10:03:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:03:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:20 smithi045 ceph-mon[95976]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:03:21.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:20 smithi045 ceph-mon[95976]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:03:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[105748]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T10:03:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:03:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[105748]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:03:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[105748]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:03:21.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[101299]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T10:03:21.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:21.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:21.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:21.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:21.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:03:21.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[101299]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:03:21.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:20 smithi032 ceph-mon[101299]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:03:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:21 smithi045 ceph-mon[95976]: pgmap v229: 73 pgs: 15 peering, 3 active+undersized, 1 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:03:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:21 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:03:22.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:21 smithi032 ceph-mon[105748]: pgmap v229: 73 pgs: 15 peering, 3 active+undersized, 1 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:03:22.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:21 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:03:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:21 smithi032 ceph-mon[101299]: pgmap v229: 73 pgs: 15 peering, 3 active+undersized, 1 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:03:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:21 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:03:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:22 smithi032 ceph-mon[101299]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-09-21T10:03:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:22 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:03:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:22 smithi032 ceph-mon[105748]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-09-21T10:03:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:22 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:03:23.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:22 smithi045 ceph-mon[95976]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-09-21T10:03:23.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:22 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:03:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:23 smithi032 ceph-mon[101299]: pgmap v230: 73 pgs: 15 peering, 2 active+undersized, 56 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:23 smithi032 ceph-mon[105748]: pgmap v230: 73 pgs: 15 peering, 2 active+undersized, 56 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:24.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:23 smithi045 ceph-mon[95976]: pgmap v230: 73 pgs: 15 peering, 2 active+undersized, 56 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:25 smithi032 ceph-mon[101299]: pgmap v231: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:25 smithi032 ceph-mon[105748]: pgmap v231: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:26.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:25 smithi045 ceph-mon[95976]: pgmap v231: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:28.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:27 smithi032 ceph-mon[105748]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:28.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:27 smithi032 ceph-mon[101299]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:27 smithi045 ceph-mon[95976]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:28.990 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-09-21T10:03:29.585 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.a smithi032 *:9093,9094 running (8m) 13s ago 14m 19.3M - 0.27.0 11f11916f8cd d6fa4f2dece8 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.a smithi045 *:3000 running (8m) 13s ago 13m 81.2M - 10.4.8 5aad1d7cf004 0ad08e43d776 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi032.fdolcm smithi032 running (12m) 13s ago 12m 20.6M - 18.2.4-871-g6a1bdf85 0adab276c97e 21fff8ad105b 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foo.smithi045.jnfsfb smithi045 running (12m) 13s ago 12m 19.6M - 18.2.4-871-g6a1bdf85 0adab276c97e 27f0b101e3bb 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.x smithi045 *:8443,9283,8765 running (9m) 13s ago 18m 459M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7ecdc0524085 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.y smithi032 *:8443,9283,8765 running (9m) 13s ago 19m 519M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 581ec7ea9f79 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a smithi032 running (7m) 13s ago 19m 56.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 525e81f3c17e 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b smithi045 running (5m) 13s ago 18m 46.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7115dfb7957a 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c smithi032 running (6m) 13s ago 18m 45.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 233d45e2730d 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.a smithi032 *:9100 running (9m) 13s ago 13m 10.5M - 1.7.0 72c9c2088986 e54d1ecd8cb3 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.b smithi045 *:9100 running (9m) 13s ago 13m 9.77M - 1.7.0 72c9c2088986 61f6c7aa590c 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (84s) 13s ago 17m 40.7M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0e29e15aee73 2024-09-21T10:03:29.587 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (51s) 13s ago 17m 46.9M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7166dcc67303 2024-09-21T10:03:29.587 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (17s) 13s ago 16m 12.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 84d2b69ff1ba 2024-09-21T10:03:29.587 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (16m) 13s ago 16m 48.9M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 208e99c75403 2024-09-21T10:03:29.587 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi045 running (16m) 13s ago 16m 56.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 81cba12364ac 2024-09-21T10:03:29.587 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi045 running (15m) 13s ago 15m 50.7M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 490d4caa0702 2024-09-21T10:03:29.587 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi045 running (15m) 13s ago 15m 49.6M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e ad56e76fa9f2 2024-09-21T10:03:29.587 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi045 running (15m) 13s ago 15m 52.9M 2481M 18.2.4-871-g6a1bdf85 0adab276c97e 8a3d61224a1b 2024-09-21T10:03:29.587 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.a smithi045 *:9095 running (8m) 13s ago 14m 51.9M - 2.51.0 1d3b7f56885b 2d9f1cee2e82 2024-09-21T10:03:29.827 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:03:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:03:29] "GET /metrics HTTP/1.1" 200 33778 "" "Prometheus/2.51.0" 2024-09-21T10:03:29.941 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:03:29.941 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-09-21T10:03:29.941 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T10:03:29.941 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 5, 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 7, 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:03:29.942 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:03:30.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:29 smithi032 ceph-mon[101299]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:30.119 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:29 smithi032 ceph-mon[105748]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:30.245 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:03:30.245 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T10:03:30.245 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-09-21T10:03:30.245 INFO:teuthology.orchestra.run.smithi032.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (1 remaining).", 2024-09-21T10:03:30.245 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-09-21T10:03:30.245 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "3/8 daemons upgraded", 2024-09-21T10:03:30.246 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:03:30.246 INFO:teuthology.orchestra.run.smithi032.stdout: "is_paused": false 2024-09-21T10:03:30.246 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:03:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:29 smithi045 ceph-mon[95976]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:31.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:30 smithi032 ceph-mon[101299]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:31.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:30 smithi032 ceph-mon[101299]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:31.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:30 smithi032 ceph-mon[101299]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:31.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:30 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2640448236' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:31.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:30 smithi032 ceph-mon[105748]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:31.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:30 smithi032 ceph-mon[105748]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:31.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:30 smithi032 ceph-mon[105748]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:31.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:30 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2640448236' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:30 smithi045 ceph-mon[95976]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:30 smithi045 ceph-mon[95976]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:30 smithi045 ceph-mon[95976]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:30 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2640448236' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:32.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:31 smithi032 ceph-mon[101299]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:31 smithi032 ceph-mon[101299]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:31 smithi032 ceph-mon[105748]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:31 smithi032 ceph-mon[105748]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:31 smithi045 ceph-mon[95976]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:31 smithi045 ceph-mon[95976]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:03:34.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:33 smithi032 ceph-mon[105748]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:33 smithi032 ceph-mon[101299]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:33 smithi045 ceph-mon[95976]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:35.194 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:03:35.195 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:03:35.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:03:36.163 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:35 smithi032 ceph-mon[105748]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:36.164 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:35 smithi032 ceph-mon[105748]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:03:36.164 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:35 smithi032 ceph-mon[105748]: Upgrade: osd.3 is safe to restart 2024-09-21T10:03:36.164 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:35 smithi032 ceph-mon[101299]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:36.164 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:35 smithi032 ceph-mon[101299]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:03:36.164 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:35 smithi032 ceph-mon[101299]: Upgrade: osd.3 is safe to restart 2024-09-21T10:03:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:35 smithi045 ceph-mon[95976]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:35 smithi045 ceph-mon[95976]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:03:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:35 smithi045 ceph-mon[95976]: Upgrade: osd.3 is safe to restart 2024-09-21T10:03:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[101299]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[101299]: Upgrade: Updating osd.3 2024-09-21T10:03:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T10:03:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[101299]: Deploying daemon osd.3 on smithi032 2024-09-21T10:03:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[105748]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[105748]: Upgrade: Updating osd.3 2024-09-21T10:03:37.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:37.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T10:03:37.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:37.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:37 smithi032 ceph-mon[105748]: Deploying daemon osd.3 on smithi032 2024-09-21T10:03:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:37 smithi045 ceph-mon[95976]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:37 smithi045 ceph-mon[95976]: Upgrade: Updating osd.3 2024-09-21T10:03:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T10:03:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:37 smithi045 ceph-mon[95976]: Deploying daemon osd.3 on smithi032 2024-09-21T10:03:39.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:39 smithi032 ceph-mon[101299]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:39.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:39 smithi032 ceph-mon[105748]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:39.457 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:39 smithi032 systemd[1]: Stopping Ceph osd.3 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:03:39.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:39 smithi045 ceph-mon[95976]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:03:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:03:39] "GET /metrics HTTP/1.1" 200 33926 "" "Prometheus/2.51.0" 2024-09-21T10:03:39.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[61571]: 2024-09-21T10:03:39.475+0000 7f17c2a12640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:03:39.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[61571]: 2024-09-21T10:03:39.475+0000 7f17c2a12640 -1 osd.3 105 *** Got signal Terminated *** 2024-09-21T10:03:39.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[61571]: 2024-09-21T10:03:39.475+0000 7f17c2a12640 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:03:40.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:40 smithi032 ceph-mon[101299]: osd.3 marked itself down and dead 2024-09-21T10:03:40.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:40 smithi032 ceph-mon[105748]: osd.3 marked itself down and dead 2024-09-21T10:03:40.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:40 smithi045 ceph-mon[95976]: osd.3 marked itself down and dead 2024-09-21T10:03:40.886 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:40 smithi032 podman[128684]: 2024-09-21 10:03:40.602415598 +0000 UTC m=+1.243028492 container died 208e99c75403556b665bfa717e92a031ca7081e19a0ca4a354d1cf9c24907b16 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T10:03:41.202 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:41 smithi032 podman[128684]: 2024-09-21 10:03:41.06710396 +0000 UTC m=+1.707716899 container cleanup 208e99c75403556b665bfa717e92a031ca7081e19a0ca4a354d1cf9c24907b16 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=reef-6a1bdf8, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T10:03:41.202 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:41 smithi032 bash[128684]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3 2024-09-21T10:03:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:41 smithi032 ceph-mon[101299]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:41 smithi032 ceph-mon[101299]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:03:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:41 smithi032 ceph-mon[101299]: osdmap e106: 8 total, 7 up, 8 in 2024-09-21T10:03:41.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:41 smithi032 ceph-mon[105748]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:41.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:41 smithi032 ceph-mon[105748]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:03:41.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:41 smithi032 ceph-mon[105748]: osdmap e106: 8 total, 7 up, 8 in 2024-09-21T10:03:41.458 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:41 smithi032 podman[128698]: 2024-09-21 10:03:41.410910056 +0000 UTC m=+0.806740677 container remove 208e99c75403556b665bfa717e92a031ca7081e19a0ca4a354d1cf9c24907b16 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-21T10:03:41.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:41 smithi045 ceph-mon[95976]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:41.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:41 smithi045 ceph-mon[95976]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:03:41.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:41 smithi045 ceph-mon[95976]: osdmap e106: 8 total, 7 up, 8 in 2024-09-21T10:03:42.058 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:41 smithi032 podman[128788]: 2024-09-21 10:03:41.651492881 +0000 UTC m=+0.016440238 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:03:42.058 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:41 smithi032 podman[128788]: 2024-09-21 10:03:41.795537465 +0000 UTC m=+0.160484810 container create b5f396d6a42223287b907b68205891a312c1730366978af45ea41cfafd8a5643 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, ceph=True, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T10:03:42.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:42 smithi032 ceph-mon[101299]: osdmap e107: 8 total, 7 up, 8 in 2024-09-21T10:03:42.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:42 smithi032 ceph-mon[105748]: osdmap e107: 8 total, 7 up, 8 in 2024-09-21T10:03:42.457 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:42 smithi032 podman[128788]: 2024-09-21 10:03:42.056704956 +0000 UTC m=+0.421652314 container init b5f396d6a42223287b907b68205891a312c1730366978af45ea41cfafd8a5643 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.build-date=20240731) 2024-09-21T10:03:42.457 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:42 smithi032 podman[128788]: 2024-09-21 10:03:42.060328894 +0000 UTC m=+0.425276240 container start b5f396d6a42223287b907b68205891a312c1730366978af45ea41cfafd8a5643 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T10:03:42.457 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:42 smithi032 podman[128788]: 2024-09-21 10:03:42.148295299 +0000 UTC m=+0.513242668 container attach b5f396d6a42223287b907b68205891a312c1730366978af45ea41cfafd8a5643 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-21T10:03:42.457 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:42 smithi032 podman[128788]: 2024-09-21 10:03:42.25890729 +0000 UTC m=+0.623854648 container died b5f396d6a42223287b907b68205891a312c1730366978af45ea41cfafd8a5643 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_BRANCH=HEAD) 2024-09-21T10:03:42.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:42 smithi045 ceph-mon[95976]: osdmap e107: 8 total, 7 up, 8 in 2024-09-21T10:03:43.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:42 smithi032 podman[128811]: 2024-09-21 10:03:42.986331461 +0000 UTC m=+0.716613276 container remove b5f396d6a42223287b907b68205891a312c1730366978af45ea41cfafd8a5643 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T10:03:43.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T10:03:43.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 systemd[1]: Stopped Ceph osd.3 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:03:43.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.3.service: Consumed 6.340s CPU time. 2024-09-21T10:03:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:43 smithi045 ceph-mon[95976]: pgmap v242: 73 pgs: 7 peering, 11 stale+active+clean, 55 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:43 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:03:43.555 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:43 smithi032 ceph-mon[101299]: pgmap v242: 73 pgs: 7 peering, 11 stale+active+clean, 55 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:43.555 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:43 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:03:43.555 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:43 smithi032 ceph-mon[105748]: pgmap v242: 73 pgs: 7 peering, 11 stale+active+clean, 55 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:43.555 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:43 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:03:43.555 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 systemd[1]: Starting Ceph osd.3 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:03:43.555 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 podman[128911]: 2024-09-21 10:03:43.445865712 +0000 UTC m=+0.113436820 container create 15c9ba7f4fe5fb1b2207b3d35fa8933c159c84d646b30ebaa7014f22bbd9974c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-21T10:03:43.556 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 podman[128911]: 2024-09-21 10:03:43.348746258 +0000 UTC m=+0.016317393 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:03:43.813 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 podman[128911]: 2024-09-21 10:03:43.673829292 +0000 UTC m=+0.341400420 container init 15c9ba7f4fe5fb1b2207b3d35fa8933c159c84d646b30ebaa7014f22bbd9974c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T10:03:43.813 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 podman[128911]: 2024-09-21 10:03:43.677307921 +0000 UTC m=+0.344879030 container start 15c9ba7f4fe5fb1b2207b3d35fa8933c159c84d646b30ebaa7014f22bbd9974c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T10:03:43.813 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 podman[128911]: 2024-09-21 10:03:43.756889349 +0000 UTC m=+0.424460466 container attach 15c9ba7f4fe5fb1b2207b3d35fa8933c159c84d646b30ebaa7014f22bbd9974c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T10:03:44.206 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate[128927]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:03:44.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 bash[128911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:03:44.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate[128927]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:03:44.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:43 smithi032 bash[128911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:03:44.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:44 smithi045 ceph-mon[95976]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:44.579 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:44 smithi032 ceph-mon[101299]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:44.579 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:44 smithi032 ceph-mon[105748]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:44.579 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate[128927]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:03:44.579 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 bash[128911]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:03:44.579 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate[128927]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T10:03:44.579 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 bash[128911]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T10:03:44.579 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate[128927]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T10:03:44.580 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 bash[128911]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T10:03:44.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate[128927]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:03:44.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 bash[128911]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:03:44.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate[128927]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:03:44.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 bash[128911]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:03:44.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate[128927]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:03:44.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 bash[128911]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:03:44.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate[128927]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T10:03:44.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 bash[128911]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T10:03:44.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:44 smithi032 podman[128911]: 2024-09-21 10:03:44.608410866 +0000 UTC m=+1.275981989 container died 15c9ba7f4fe5fb1b2207b3d35fa8933c159c84d646b30ebaa7014f22bbd9974c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:03:45.397 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:45 smithi032 ceph-mon[105748]: pgmap v243: 73 pgs: 16 active+undersized, 7 peering, 3 stale+active+clean, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-21T10:03:45.397 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:45 smithi032 ceph-mon[101299]: pgmap v243: 73 pgs: 16 active+undersized, 7 peering, 3 stale+active+clean, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-21T10:03:45.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:45 smithi045 ceph-mon[95976]: pgmap v243: 73 pgs: 16 active+undersized, 7 peering, 3 stale+active+clean, 2 active+undersized+degraded, 45 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-21T10:03:45.707 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:45 smithi032 podman[129063]: 2024-09-21 10:03:45.395230073 +0000 UTC m=+0.780073307 container remove 15c9ba7f4fe5fb1b2207b3d35fa8933c159c84d646b30ebaa7014f22bbd9974c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:03:45.707 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:45 smithi032 podman[129123]: 2024-09-21 10:03:45.613052107 +0000 UTC m=+0.113925324 container create e5f8912f154b3c1b70f73fb4ca1c8173e1d68acd8ec845298299d6742261015b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T10:03:45.707 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:45 smithi032 podman[129123]: 2024-09-21 10:03:45.515930733 +0000 UTC m=+0.016803916 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:03:46.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:45 smithi032 podman[129123]: 2024-09-21 10:03:45.848975743 +0000 UTC m=+0.349848921 container init e5f8912f154b3c1b70f73fb4ca1c8173e1d68acd8ec845298299d6742261015b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T10:03:46.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:45 smithi032 podman[129123]: 2024-09-21 10:03:45.852245467 +0000 UTC m=+0.353118639 container start e5f8912f154b3c1b70f73fb4ca1c8173e1d68acd8ec845298299d6742261015b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T10:03:46.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:45 smithi032 ceph-osd[129141]: -- 172.21.15.32:0/2840765644 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c5355b3a40 con 0x55c536382400 2024-09-21T10:03:46.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:45 smithi032 bash[129123]: e5f8912f154b3c1b70f73fb4ca1c8173e1d68acd8ec845298299d6742261015b 2024-09-21T10:03:46.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:45 smithi032 systemd[1]: Started Ceph osd.3 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:03:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[105748]: pgmap v244: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:47.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:47.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:46 smithi032 ceph-mon[101299]: pgmap v244: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:46 smithi045 ceph-mon[95976]: pgmap v244: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:47.838 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[129137]: 2024-09-21T10:03:47.482+0000 7ff69157b640 -1 Falling back to public interface 2024-09-21T10:03:48.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:47 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T10:03:48.146 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:48 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T10:03:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:48 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T10:03:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:49 smithi045 ceph-mon[95976]: pgmap v245: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:49.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:03:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:03:49] "GET /metrics HTTP/1.1" 200 33926 "" "Prometheus/2.51.0" 2024-09-21T10:03:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:49 smithi032 ceph-mon[101299]: pgmap v245: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:49 smithi032 ceph-mon[105748]: pgmap v245: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:50.788 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:50.788 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:50.788 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:50.788 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:50.789 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:50.789 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:50.789 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:50.789 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:51.206 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[129137]: 2024-09-21T10:03:51.031+0000 7ff69157b640 -1 osd.3 105 log_to_monitors true 2024-09-21T10:03:51.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-mon[101299]: pgmap v246: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:51.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-mon[101299]: from='osd.3 [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:03:51.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-mon[101299]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:03:51.801 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:51.801 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-mon[105748]: pgmap v246: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:51.802 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-mon[105748]: from='osd.3 [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:03:51.802 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-mon[105748]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:03:51.802 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:51 smithi045 ceph-mon[95976]: pgmap v246: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:51 smithi045 ceph-mon[95976]: from='osd.3 [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:03:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:51 smithi045 ceph-mon[95976]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:03:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:51 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:52.206 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:03:51 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[129137]: 2024-09-21T10:03:51.845+0000 7ff68531d640 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:03:52.926 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[105748]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:03:52.926 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[105748]: osdmap e108: 8 total, 7 up, 8 in 2024-09-21T10:03:52.926 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[105748]: from='osd.3 [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:03:52.926 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[105748]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:03:52.926 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[105748]: from='osd.3 ' entity='osd.3' 2024-09-21T10:03:52.926 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[101299]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:03:52.926 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[101299]: osdmap e108: 8 total, 7 up, 8 in 2024-09-21T10:03:52.926 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[101299]: from='osd.3 [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:03:52.927 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[101299]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:03:52.927 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:52 smithi032 ceph-mon[101299]: from='osd.3 ' entity='osd.3' 2024-09-21T10:03:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:52 smithi045 ceph-mon[95976]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:03:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:52 smithi045 ceph-mon[95976]: osdmap e108: 8 total, 7 up, 8 in 2024-09-21T10:03:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:52 smithi045 ceph-mon[95976]: from='osd.3 [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:03:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:52 smithi045 ceph-mon[95976]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:03:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:52 smithi045 ceph-mon[95976]: from='osd.3 ' entity='osd.3' 2024-09-21T10:03:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: pgmap v248: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:03:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: osd.3 [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087] boot 2024-09-21T10:03:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T10:03:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:03:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:03:53.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:53.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: pgmap v248: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: osd.3 [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087] boot 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:03:53.962 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:03:53.963 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:53.964 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: pgmap v248: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-21T10:03:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: osd.3 [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087] boot 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:54.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:03:54.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:54.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:54.039 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:03:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:54 smithi032 ceph-mon[105748]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:03:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:54 smithi032 ceph-mon[105748]: Upgrade: Finalizing container_image settings 2024-09-21T10:03:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:54 smithi032 ceph-mon[105748]: Upgrade: Complete! 2024-09-21T10:03:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:54 smithi032 ceph-mon[105748]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T10:03:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:54 smithi032 ceph-mon[101299]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:03:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:54 smithi032 ceph-mon[101299]: Upgrade: Finalizing container_image settings 2024-09-21T10:03:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:54 smithi032 ceph-mon[101299]: Upgrade: Complete! 2024-09-21T10:03:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:54 smithi032 ceph-mon[101299]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T10:03:55.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:54 smithi045 ceph-mon[95976]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:03:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:54 smithi045 ceph-mon[95976]: Upgrade: Finalizing container_image settings 2024-09-21T10:03:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:54 smithi045 ceph-mon[95976]: Upgrade: Complete! 2024-09-21T10:03:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:54 smithi045 ceph-mon[95976]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T10:03:56.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:55 smithi032 ceph-mon[101299]: pgmap v251: 73 pgs: 21 active+undersized, 2 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:03:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:55 smithi032 ceph-mon[105748]: pgmap v251: 73 pgs: 21 active+undersized, 2 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:03:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:55 smithi045 ceph-mon[95976]: pgmap v251: 73 pgs: 21 active+undersized, 2 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:03:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:56 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:56 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:56 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:58.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:57 smithi032 ceph-mon[105748]: pgmap v252: 73 pgs: 13 active+undersized, 2 active+undersized+degraded, 58 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:03:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:57 smithi032 ceph-mon[101299]: pgmap v252: 73 pgs: 13 active+undersized, 2 active+undersized+degraded, 58 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:03:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:57 smithi045 ceph-mon[95976]: pgmap v252: 73 pgs: 13 active+undersized, 2 active+undersized+degraded, 58 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-21T10:03:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:58 smithi032 ceph-mon[101299]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:58 smithi032 ceph-mon[101299]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-09-21T10:03:59.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:03:58 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:03:59.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:58 smithi032 ceph-mon[105748]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:59.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:58 smithi032 ceph-mon[105748]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-09-21T10:03:59.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:03:58 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:03:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:58 smithi045 ceph-mon[95976]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:03:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:58 smithi045 ceph-mon[95976]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-09-21T10:03:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:03:58 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:03:59.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:03:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:03:59] "GET /metrics HTTP/1.1" 200 33926 "" "Prometheus/2.51.0" 2024-09-21T10:04:01.215 DEBUG:teuthology.parallel:result is None 2024-09-21T10:04:01.215 INFO:teuthology.run_tasks:Running task print... 2024-09-21T10:04:01.225 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-21T10:04:01.225 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T10:04:01.244 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T10:04:01.244 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T10:04:01.244 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T10:04:01.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:01 smithi032 ceph-mon[101299]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:01.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:01 smithi032 ceph-mon[101299]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:04:01.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:01 smithi032 ceph-mon[105748]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:01.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:01 smithi032 ceph-mon[105748]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:04:01.460 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:01.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:01 smithi045 ceph-mon[95976]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:01.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:01 smithi045 ceph-mon[95976]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:04:03.306 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:04:03.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:03 smithi032 ceph-mon[101299]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:03 smithi032 ceph-mon[105748]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:03.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:03 smithi045 ceph-mon[95976]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:03.928 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":40,"flags":0,"active_gid":25736,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":1713386516},{"type":"v1","addr":"172.21.15.32:6801","nonce":1713386516}]},"active_addr":"172.21.15.32:6801/1713386516","active_change":"2024-09-21T09:56:17.981247+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":25396,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.8","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.8","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.32:8443/","prometheus":"http://172.21.15.32:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":90,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":4235638897}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1614916298}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":4088159811}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1835888717}]}]} 2024-09-21T10:04:03.933 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T10:04:03.933 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T10:04:03.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:04.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:04 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2058355771' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-21T10:04:04.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:04.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:04.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:04.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:04 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2058355771' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-21T10:04:04.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:04.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:04.428 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:04 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2058355771' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-21T10:04:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:05.784 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:05 smithi032 ceph-mon[105748]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:05.784 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:05 smithi032 ceph-mon[101299]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:05 smithi045 ceph-mon[95976]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:05.975 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:04:05.976 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":110,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:43:37.317307+0000","modified":"2024-09-21T10:03:53.826703+0000","last_up_change":"2024-09-21T10:03:52.822594+0000","last_in_change":"2024-09-21T09:48:18.335792+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:46:40.328691+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:49:08.740859+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:50:21.329505+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:50:21.832696+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0f6f5dda-c134-4283-a91d-cf501ef19062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":4212241103},{"type":"v1","addr":"172.21.15.32:6803","nonce":4212241103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":4212241103},{"type":"v1","addr":"172.21.15.32:6805","nonce":4212241103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":4212241103},{"type":"v1","addr":"172.21.15.32:6809","nonce":4212241103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":4212241103},{"type":"v1","addr":"172.21.15.32:6807","nonce":4212241103}]},"public_addr":"172.21.15.32:6803/4212241103","cluster_addr":"172.21.15.32:6805/4212241103","heartbeat_back_addr":"172.21.15.32:6809/4212241103","heartbeat_front_addr":"172.21.15.32:6807/4212241103","state":["exists","up"]},{"osd":1,"uuid":"bdaa43fd-b969-4d06-b6eb-97b415c4e118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6811","nonce":1357153778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6813","nonce":1357153778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6817","nonce":1357153778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6815","nonce":1357153778}]},"public_addr":"172.21.15.32:6811/1357153778","cluster_addr":"172.21.15.32:6813/1357153778","heartbeat_back_addr":"172.21.15.32:6817/1357153778","heartbeat_front_addr":"172.21.15.32:6815/1357153778","state":["exists","up"]},{"osd":2,"uuid":"1f12d9f2-90b0-44f6-8432-0b398194b3b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6819","nonce":3097218490}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6821","nonce":3097218490}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6825","nonce":3097218490}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6823","nonce":3097218490}]},"public_addr":"172.21.15.32:6819/3097218490","cluster_addr":"172.21.15.32:6821/3097218490","heartbeat_back_addr":"172.21.15.32:6825/3097218490","heartbeat_front_addr":"172.21.15.32:6823/3097218490","state":["exists","up"]},{"osd":3,"uuid":"6a568fd5-5fd9-46f8-b3af-9fa196ed617b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6827","nonce":438066087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6829","nonce":438066087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6833","nonce":438066087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6831","nonce":438066087}]},"public_addr":"172.21.15.32:6827/438066087","cluster_addr":"172.21.15.32:6829/438066087","heartbeat_back_addr":"172.21.15.32:6833/438066087","heartbeat_front_addr":"172.21.15.32:6831/438066087","state":["exists","up"]},{"osd":4,"uuid":"2cf149e0-4c2f-46eb-9ca2-a51907aea3b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6801","nonce":413300252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6803","nonce":413300252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6807","nonce":413300252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6805","nonce":413300252}]},"public_addr":"172.21.15.45:6801/413300252","cluster_addr":"172.21.15.45:6803/413300252","heartbeat_back_addr":"172.21.15.45:6807/413300252","heartbeat_front_addr":"172.21.15.45:6805/413300252","state":["exists","up"]},{"osd":5,"uuid":"a6c78855-6756-4b0b-9a9c-8d102079540b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6809","nonce":3168849729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6811","nonce":3168849729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6815","nonce":3168849729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6813","nonce":3168849729}]},"public_addr":"172.21.15.45:6809/3168849729","cluster_addr":"172.21.15.45:6811/3168849729","heartbeat_back_addr":"172.21.15.45:6815/3168849729","heartbeat_front_addr":"172.21.15.45:6813/3168849729","state":["exists","up"]},{"osd":6,"uuid":"d476169d-dffc-4f1e-b60b-4f836f53285e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6817","nonce":2158139679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6819","nonce":2158139679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6823","nonce":2158139679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6821","nonce":2158139679}]},"public_addr":"172.21.15.45:6817/2158139679","cluster_addr":"172.21.15.45:6819/2158139679","heartbeat_back_addr":"172.21.15.45:6823/2158139679","heartbeat_front_addr":"172.21.15.45:6821/2158139679","state":["exists","up"]},{"osd":7,"uuid":"021a4fd0-f770-4799-9683-b0aa7cd593bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6825","nonce":3336053350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6827","nonce":3336053350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6831","nonce":3336053350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6829","nonce":3336053350}]},"public_addr":"172.21.15.45:6825/3336053350","cluster_addr":"172.21.15.45:6827/3336053350","heartbeat_back_addr":"172.21.15.45:6831/3336053350","heartbeat_front_addr":"172.21.15.45:6829/3336053350","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:50.697928+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:13.831263+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:37.360357+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:00.010474+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:21.607129+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:45.583237+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:09.486296+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:33.456690+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/3511996666":"2024-09-22T09:56:17.980985+0000","172.21.15.32:6800/2805220519":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1807347540":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1200491506":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2408427509":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1178553739":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6801/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.45:0/2114035806":"2024-09-22T09:53:38.533217+0000","172.21.15.32:6801/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.45:0/102537314":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/458122323":"2024-09-22T09:49:07.125921+0000","172.21.15.45:6835/1148645562":"2024-09-22T09:53:38.533217+0000","172.21.15.45:0/599831880":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/1465894634":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/230160407":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2785183325":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/431176591":"2024-09-22T09:53:08.870463+0000","172.21.15.32:6800/431176591":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2103196055":"2024-09-22T09:53:08.870463+0000","172.21.15.32:6801/2805220519":"2024-09-22T09:56:17.980985+0000","172.21.15.32:6800/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/2042031504":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/1259145858":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/533058733":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/1858508442":"2024-09-22T09:53:08.870463+0000","172.21.15.32:6801/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/2895073807":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.45:0/338006180":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/2388392475":"2024-09-22T09:49:07.125921+0000","172.21.15.45:6834/1148645562":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/1102447792":"2024-09-22T09:56:17.980985+0000","172.21.15.45:0/2100270146":"2024-09-22T09:53:38.533217+0000","172.21.15.45:0/431361432":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/3332221400":"2024-09-22T09:43:58.900171+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:04:06.604 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T10:04:06.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:04:06.823 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:06.944 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/686866114' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:04:06.944 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/686866114' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:04:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/686866114' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:04:07.930 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:07 smithi032 ceph-mon[101299]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:07.930 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:07 smithi032 ceph-mon[105748]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:08.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:07 smithi045 ceph-mon[95976]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:08.627 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:04:08.627 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":110,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:43:37.317307+0000","modified":"2024-09-21T10:03:53.826703+0000","last_up_change":"2024-09-21T10:03:52.822594+0000","last_in_change":"2024-09-21T09:48:18.335792+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:46:40.328691+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:49:08.740859+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:50:21.329505+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:50:21.832696+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0f6f5dda-c134-4283-a91d-cf501ef19062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":4212241103},{"type":"v1","addr":"172.21.15.32:6803","nonce":4212241103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":4212241103},{"type":"v1","addr":"172.21.15.32:6805","nonce":4212241103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":4212241103},{"type":"v1","addr":"172.21.15.32:6809","nonce":4212241103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":4212241103},{"type":"v1","addr":"172.21.15.32:6807","nonce":4212241103}]},"public_addr":"172.21.15.32:6803/4212241103","cluster_addr":"172.21.15.32:6805/4212241103","heartbeat_back_addr":"172.21.15.32:6809/4212241103","heartbeat_front_addr":"172.21.15.32:6807/4212241103","state":["exists","up"]},{"osd":1,"uuid":"bdaa43fd-b969-4d06-b6eb-97b415c4e118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6811","nonce":1357153778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6813","nonce":1357153778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6817","nonce":1357153778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6815","nonce":1357153778}]},"public_addr":"172.21.15.32:6811/1357153778","cluster_addr":"172.21.15.32:6813/1357153778","heartbeat_back_addr":"172.21.15.32:6817/1357153778","heartbeat_front_addr":"172.21.15.32:6815/1357153778","state":["exists","up"]},{"osd":2,"uuid":"1f12d9f2-90b0-44f6-8432-0b398194b3b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6819","nonce":3097218490}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6821","nonce":3097218490}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6825","nonce":3097218490}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6823","nonce":3097218490}]},"public_addr":"172.21.15.32:6819/3097218490","cluster_addr":"172.21.15.32:6821/3097218490","heartbeat_back_addr":"172.21.15.32:6825/3097218490","heartbeat_front_addr":"172.21.15.32:6823/3097218490","state":["exists","up"]},{"osd":3,"uuid":"6a568fd5-5fd9-46f8-b3af-9fa196ed617b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6827","nonce":438066087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6829","nonce":438066087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6833","nonce":438066087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6831","nonce":438066087}]},"public_addr":"172.21.15.32:6827/438066087","cluster_addr":"172.21.15.32:6829/438066087","heartbeat_back_addr":"172.21.15.32:6833/438066087","heartbeat_front_addr":"172.21.15.32:6831/438066087","state":["exists","up"]},{"osd":4,"uuid":"2cf149e0-4c2f-46eb-9ca2-a51907aea3b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6801","nonce":413300252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6803","nonce":413300252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6807","nonce":413300252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6805","nonce":413300252}]},"public_addr":"172.21.15.45:6801/413300252","cluster_addr":"172.21.15.45:6803/413300252","heartbeat_back_addr":"172.21.15.45:6807/413300252","heartbeat_front_addr":"172.21.15.45:6805/413300252","state":["exists","up"]},{"osd":5,"uuid":"a6c78855-6756-4b0b-9a9c-8d102079540b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6809","nonce":3168849729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6811","nonce":3168849729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6815","nonce":3168849729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":3168849729},{"type":"v1","addr":"172.21.15.45:6813","nonce":3168849729}]},"public_addr":"172.21.15.45:6809/3168849729","cluster_addr":"172.21.15.45:6811/3168849729","heartbeat_back_addr":"172.21.15.45:6815/3168849729","heartbeat_front_addr":"172.21.15.45:6813/3168849729","state":["exists","up"]},{"osd":6,"uuid":"d476169d-dffc-4f1e-b60b-4f836f53285e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6817","nonce":2158139679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6819","nonce":2158139679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6823","nonce":2158139679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6821","nonce":2158139679}]},"public_addr":"172.21.15.45:6817/2158139679","cluster_addr":"172.21.15.45:6819/2158139679","heartbeat_back_addr":"172.21.15.45:6823/2158139679","heartbeat_front_addr":"172.21.15.45:6821/2158139679","state":["exists","up"]},{"osd":7,"uuid":"021a4fd0-f770-4799-9683-b0aa7cd593bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6825","nonce":3336053350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6827","nonce":3336053350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6831","nonce":3336053350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6829","nonce":3336053350}]},"public_addr":"172.21.15.45:6825/3336053350","cluster_addr":"172.21.15.45:6827/3336053350","heartbeat_back_addr":"172.21.15.45:6831/3336053350","heartbeat_front_addr":"172.21.15.45:6829/3336053350","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:50.697928+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:13.831263+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:37.360357+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:00.010474+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:21.607129+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:45.583237+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:09.486296+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:33.456690+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/3511996666":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1200491506":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/1465894634":"2024-09-22T09:53:08.870463+0000","172.21.15.32:6801/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.45:0/2114035806":"2024-09-22T09:53:38.533217+0000","172.21.15.32:6801/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.45:0/102537314":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/458122323":"2024-09-22T09:49:07.125921+0000","172.21.15.45:6835/1148645562":"2024-09-22T09:53:38.533217+0000","172.21.15.45:0/599831880":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/1807347540":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/230160407":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2785183325":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/431176591":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2408427509":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1178553739":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/431176591":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2103196055":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2042031504":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/1259145858":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/533058733":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/3332221400":"2024-09-22T09:43:58.900171+0000","172.21.15.45:0/2100270146":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/1102447792":"2024-09-22T09:56:17.980985+0000","172.21.15.45:0/431361432":"2024-09-22T09:53:38.533217+0000","172.21.15.32:6800/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/2805220519":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1858508442":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2895073807":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/2805220519":"2024-09-22T09:56:17.980985+0000","172.21.15.45:0/338006180":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/2388392475":"2024-09-22T09:49:07.125921+0000","172.21.15.45:6834/1148645562":"2024-09-22T09:53:38.533217+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:04:09.262 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T10:04:09.262 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T10:04:09.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T10:04:09.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T10:04:09.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T10:04:09.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T10:04:09.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T10:04:09.265 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T10:04:09.537 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:04:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:04:09] "GET /metrics HTTP/1.1" 200 33926 "" "Prometheus/2.51.0" 2024-09-21T10:04:09.687 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:09.701 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:09.701 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:09.701 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:09.702 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:09.709 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:09.709 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:09.723 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:09.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:09 smithi032 ceph-mon[101299]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:09.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3399678282' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:04:09.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:09 smithi032 ceph-mon[105748]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:09.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3399678282' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:04:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:09 smithi045 ceph-mon[95976]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3399678282' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:04:11.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:11 smithi032 ceph-mon[101299]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:11.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:11 smithi032 ceph-mon[105748]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:11 smithi045 ceph-mon[95976]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:13.611 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:13 smithi032 ceph-mon[105748]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:13.612 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:13 smithi032 ceph-mon[101299]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:13.837 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:13 smithi045 ceph-mon[95976]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:15.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:15 smithi032 ceph-mon[105748]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:15.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:15 smithi032 ceph-mon[101299]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:15 smithi045 ceph-mon[95976]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:17 smithi032 ceph-mon[101299]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:17 smithi032 ceph-mon[105748]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:17 smithi045 ceph-mon[95976]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:18.599 INFO:teuthology.orchestra.run.smithi032.stdout:231928234175 2024-09-21T10:04:18.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T10:04:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:18.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:19.904 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:19 smithi032 ceph-mon[101299]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:19.904 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:19 smithi032 ceph-mon[105748]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:19.904 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:04:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:04:19] "GET /metrics HTTP/1.1" 200 33930 "" "Prometheus/2.51.0" 2024-09-21T10:04:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:19 smithi045 ceph-mon[95976]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:21.135 INFO:teuthology.orchestra.run.smithi032.stdout:446676598798 2024-09-21T10:04:21.135 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T10:04:21.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:21 smithi032 ceph-mon[105748]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:21 smithi032 ceph-mon[101299]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:21 smithi045 ceph-mon[95976]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:22.345 INFO:teuthology.orchestra.run.smithi032.stdout:197568495814 2024-09-21T10:04:22.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T10:04:22.719 INFO:teuthology.orchestra.run.smithi032.stdout:468151435271 2024-09-21T10:04:22.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T10:04:23.537 INFO:teuthology.orchestra.run.smithi032.stdout:403726925852 2024-09-21T10:04:23.537 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T10:04:23.859 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:23 smithi032 ceph-mon[101299]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:23.859 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:23 smithi032 ceph-mon[105748]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:23 smithi045 ceph-mon[95976]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:24.607 INFO:teuthology.orchestra.run.smithi032.stdout:128849019087 2024-09-21T10:04:24.607 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T10:04:25.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:25 smithi032 ceph-mon[101299]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:25.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:25 smithi032 ceph-mon[105748]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:26.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:25 smithi045 ceph-mon[95976]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:26.837 INFO:teuthology.orchestra.run.smithi032.stdout:425201762326 2024-09-21T10:04:26.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T10:04:27.239 INFO:teuthology.orchestra.run.smithi032.stdout:163208757452 2024-09-21T10:04:27.239 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T10:04:27.353 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:27.354 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:27.361 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:27.363 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:27.363 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:27.381 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:27.391 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:27.801 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:27 smithi032 ceph-mon[105748]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:27.802 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:27 smithi032 ceph-mon[101299]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:27 smithi045 ceph-mon[95976]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:29.798 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:29 smithi032 ceph-mon[101299]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:29.799 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:29 smithi032 ceph-mon[105748]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:29.799 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:04:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:04:29] "GET /metrics HTTP/1.1" 200 33930 "" "Prometheus/2.51.0" 2024-09-21T10:04:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:29 smithi045 ceph-mon[95976]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:31.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:31 smithi032 ceph-mon[101299]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:31.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:31 smithi032 ceph-mon[105748]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:32.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:31 smithi045 ceph-mon[95976]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:33.296 INFO:teuthology.orchestra.run.smithi032.stdout:468151435274 2024-09-21T10:04:33.868 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:33 smithi032 ceph-mon[105748]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:33.869 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:33 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/797299479' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:04:33.869 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:33.869 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:33 smithi032 ceph-mon[101299]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:33.869 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:33 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/797299479' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:04:33.869 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:33.891 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435271 got 468151435274 for osd.3 2024-09-21T10:04:33.892 DEBUG:teuthology.parallel:result is None 2024-09-21T10:04:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:33 smithi045 ceph-mon[95976]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:33 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/797299479' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:04:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:34.272 INFO:teuthology.orchestra.run.smithi032.stdout:231928234178 2024-09-21T10:04:34.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/217822232' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:04:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/217822232' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:04:35.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/217822232' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:04:35.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:35 smithi032 ceph-mon[101299]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:35.833 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:35 smithi032 ceph-mon[105748]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:36.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:35 smithi045 ceph-mon[95976]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:36.572 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234175 got 231928234178 for osd.7 2024-09-21T10:04:36.572 DEBUG:teuthology.parallel:result is None 2024-09-21T10:04:37.377 INFO:teuthology.orchestra.run.smithi032.stdout:403726925855 2024-09-21T10:04:37.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:37 smithi032 ceph-mon[101299]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:37.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:37 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1925969068' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:04:37.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:37 smithi032 ceph-mon[105748]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:37.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:37 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1925969068' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:04:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:37 smithi045 ceph-mon[95976]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:37 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1925969068' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:04:38.306 INFO:teuthology.orchestra.run.smithi032.stdout:197568495817 2024-09-21T10:04:38.513 INFO:teuthology.orchestra.run.smithi032.stdout:128849019090 2024-09-21T10:04:38.854 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:38 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2536354570' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:04:38.855 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:38 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2101466758' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:04:38.855 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:38 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2536354570' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:04:38.855 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:38 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2101466758' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:04:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:38 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2536354570' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:04:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:38 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2101466758' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:04:39.139 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495814 got 197568495817 for osd.6 2024-09-21T10:04:39.139 DEBUG:teuthology.parallel:result is None 2024-09-21T10:04:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:39 smithi032 ceph-mon[101299]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:39.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:39 smithi032 ceph-mon[105748]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:39.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:04:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:04:39] "GET /metrics HTTP/1.1" 200 33925 "" "Prometheus/2.51.0" 2024-09-21T10:04:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:39 smithi045 ceph-mon[95976]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:40.159 INFO:teuthology.orchestra.run.smithi032.stdout:425201762328 2024-09-21T10:04:40.755 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:40 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/836707017' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:04:40.755 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:40 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/836707017' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:04:40.756 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925852 got 403726925855 for osd.0 2024-09-21T10:04:40.756 DEBUG:teuthology.parallel:result is None 2024-09-21T10:04:41.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:40 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/836707017' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:04:41.281 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019087 got 128849019090 for osd.4 2024-09-21T10:04:41.281 DEBUG:teuthology.parallel:result is None 2024-09-21T10:04:41.403 INFO:teuthology.orchestra.run.smithi032.stdout:446676598802 2024-09-21T10:04:41.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:41 smithi032 ceph-mon[101299]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:41 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3370597970' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:04:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:41 smithi032 ceph-mon[105748]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:41 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3370597970' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:04:42.030 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:41 smithi045 ceph-mon[95976]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:41 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3370597970' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:04:42.041 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676598798 got 446676598802 for osd.2 2024-09-21T10:04:42.041 DEBUG:teuthology.parallel:result is None 2024-09-21T10:04:42.475 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762326 got 425201762328 for osd.1 2024-09-21T10:04:42.475 DEBUG:teuthology.parallel:result is None 2024-09-21T10:04:43.867 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:43 smithi032 ceph-mon[105748]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:43.867 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:43 smithi032 ceph-mon[101299]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:43 smithi045 ceph-mon[95976]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:44.264 INFO:teuthology.orchestra.run.smithi032.stdout:163208757455 2024-09-21T10:04:44.874 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:44 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2994596740' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:04:44.875 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:44 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2994596740' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:04:44.898 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757452 got 163208757455 for osd.5 2024-09-21T10:04:44.898 DEBUG:teuthology.parallel:result is None 2024-09-21T10:04:44.898 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T10:04:44.898 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:04:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:44 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2994596740' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:04:45.109 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:45.916 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:45 smithi032 ceph-mon[105748]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:45.917 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:45 smithi032 ceph-mon[101299]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:45 smithi045 ceph-mon[95976]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:46.963 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:04:46.964 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:04:47.593 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":277,"stamp":"2024-09-21T10:04:46.122221+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":262452,"kb_used_data":23596,"kb_used_omap":35,"kb_used_meta":238684,"kb_avail":749698764,"statfs":{"total":767960285184,"available":767691534336,"internally_reserved":0,"allocated":24162304,"data_stored":19392999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":244413134},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002636"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843669+0000","last_change":"2024-09-21T10:03:53.843548+0000","last_active":"2024-09-21T10:03:53.843669+0000","last_peered":"2024-09-21T10:03:53.843669+0000","last_clean":"2024-09-21T10:03:53.843669+0000","last_became_active":"2024-09-21T10:03:53.843182+0000","last_became_peered":"2024-09-21T10:03:53.843182+0000","last_unstale":"2024-09-21T10:03:53.843669+0000","last_undegraded":"2024-09-21T10:03:53.843669+0000","last_fullsized":"2024-09-21T10:03:53.843669+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:09.464209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834897+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:03:53.834897+0000","last_peered":"2024-09-21T10:03:53.834897+0000","last_clean":"2024-09-21T10:03:53.834897+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:03:53.834897+0000","last_undegraded":"2024-09-21T10:03:53.834897+0000","last_fullsized":"2024-09-21T10:03:53.834897+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839987+0000","last_change":"2024-09-21T10:03:53.839987+0000","last_active":"2024-09-21T10:03:53.839987+0000","last_peered":"2024-09-21T10:03:53.839987+0000","last_clean":"2024-09-21T10:03:53.839987+0000","last_became_active":"2024-09-21T10:03:53.839664+0000","last_became_peered":"2024-09-21T10:03:53.839664+0000","last_unstale":"2024-09-21T10:03:53.839987+0000","last_undegraded":"2024-09-21T10:03:53.839987+0000","last_fullsized":"2024-09-21T10:03:53.839987+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:51.726392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837862+0000","last_change":"2024-09-21T10:03:53.837496+0000","last_active":"2024-09-21T10:03:53.837862+0000","last_peered":"2024-09-21T10:03:53.837862+0000","last_clean":"2024-09-21T10:03:53.837862+0000","last_became_active":"2024-09-21T10:03:53.837166+0000","last_became_peered":"2024-09-21T10:03:53.837166+0000","last_unstale":"2024-09-21T10:03:53.837862+0000","last_undegraded":"2024-09-21T10:03:53.837862+0000","last_fullsized":"2024-09-21T10:03:53.837862+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:45.795571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845452+0000","last_change":"2024-09-21T10:03:53.844990+0000","last_active":"2024-09-21T10:03:53.845452+0000","last_peered":"2024-09-21T10:03:53.845452+0000","last_clean":"2024-09-21T10:03:53.845452+0000","last_became_active":"2024-09-21T10:03:53.844664+0000","last_became_peered":"2024-09-21T10:03:53.844664+0000","last_unstale":"2024-09-21T10:03:53.845452+0000","last_undegraded":"2024-09-21T10:03:53.845452+0000","last_fullsized":"2024-09-21T10:03:53.845452+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:01.031522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835247+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:03:53.835247+0000","last_peered":"2024-09-21T10:03:53.835247+0000","last_clean":"2024-09-21T10:03:53.835247+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:03:53.835247+0000","last_undegraded":"2024-09-21T10:03:53.835247+0000","last_fullsized":"2024-09-21T10:03:53.835247+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845872+0000","last_change":"2024-09-21T10:03:53.845785+0000","last_active":"2024-09-21T10:03:53.845872+0000","last_peered":"2024-09-21T10:03:53.845872+0000","last_clean":"2024-09-21T10:03:53.845872+0000","last_became_active":"2024-09-21T10:03:53.845303+0000","last_became_peered":"2024-09-21T10:03:53.845303+0000","last_unstale":"2024-09-21T10:03:53.845872+0000","last_undegraded":"2024-09-21T10:03:53.845872+0000","last_fullsized":"2024-09-21T10:03:53.845872+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:04.976007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837943+0000","last_change":"2024-09-21T10:02:12.838786+0000","last_active":"2024-09-21T10:03:53.837943+0000","last_peered":"2024-09-21T10:03:53.837943+0000","last_clean":"2024-09-21T10:03:53.837943+0000","last_became_active":"2024-09-21T10:02:12.838242+0000","last_became_peered":"2024-09-21T10:02:12.838242+0000","last_unstale":"2024-09-21T10:03:53.837943+0000","last_undegraded":"2024-09-21T10:03:53.837943+0000","last_fullsized":"2024-09-21T10:03:53.837943+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:31.071136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835170+0000","last_change":"2024-09-21T10:02:12.840334+0000","last_active":"2024-09-21T10:03:53.835170+0000","last_peered":"2024-09-21T10:03:53.835170+0000","last_clean":"2024-09-21T10:03:53.835170+0000","last_became_active":"2024-09-21T10:02:12.839928+0000","last_became_peered":"2024-09-21T10:02:12.839928+0000","last_unstale":"2024-09-21T10:03:53.835170+0000","last_undegraded":"2024-09-21T10:03:53.835170+0000","last_fullsized":"2024-09-21T10:03:53.835170+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:38.593644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836136+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T10:03:52.836136+0000","last_peered":"2024-09-21T10:03:52.836136+0000","last_clean":"2024-09-21T10:03:52.836136+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T10:03:52.836136+0000","last_undegraded":"2024-09-21T10:03:52.836136+0000","last_fullsized":"2024-09-21T10:03:52.836136+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.836116+0000","last_change":"2024-09-21T10:02:46.662169+0000","last_active":"2024-09-21T10:03:53.836116+0000","last_peered":"2024-09-21T10:03:53.836116+0000","last_clean":"2024-09-21T10:03:53.836116+0000","last_became_active":"2024-09-21T10:02:46.661721+0000","last_became_peered":"2024-09-21T10:02:46.661721+0000","last_unstale":"2024-09-21T10:03:53.836116+0000","last_undegraded":"2024-09-21T10:03:53.836116+0000","last_fullsized":"2024-09-21T10:03:53.836116+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:10.414152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840738+0000","last_change":"2024-09-21T10:03:53.840738+0000","last_active":"2024-09-21T10:03:53.840738+0000","last_peered":"2024-09-21T10:03:53.840738+0000","last_clean":"2024-09-21T10:03:53.840738+0000","last_became_active":"2024-09-21T10:03:53.840363+0000","last_became_peered":"2024-09-21T10:03:53.840363+0000","last_unstale":"2024-09-21T10:03:53.840738+0000","last_undegraded":"2024-09-21T10:03:53.840738+0000","last_fullsized":"2024-09-21T10:03:53.840738+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:26.371181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835739+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T10:03:53.835739+0000","last_peered":"2024-09-21T10:03:53.835739+0000","last_clean":"2024-09-21T10:03:53.835739+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T10:03:53.835739+0000","last_undegraded":"2024-09-21T10:03:53.835739+0000","last_fullsized":"2024-09-21T10:03:53.835739+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833635+0000","last_change":"2024-09-21T10:02:46.663127+0000","last_active":"2024-09-21T10:03:52.833635+0000","last_peered":"2024-09-21T10:03:52.833635+0000","last_clean":"2024-09-21T10:03:52.833635+0000","last_became_active":"2024-09-21T10:02:46.662791+0000","last_became_peered":"2024-09-21T10:02:46.662791+0000","last_unstale":"2024-09-21T10:03:52.833635+0000","last_undegraded":"2024-09-21T10:03:52.833635+0000","last_fullsized":"2024-09-21T10:03:52.833635+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:59.990122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843912+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:03:53.843912+0000","last_peered":"2024-09-21T10:03:53.843912+0000","last_clean":"2024-09-21T10:03:53.843912+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:03:53.843912+0000","last_undegraded":"2024-09-21T10:03:53.843912+0000","last_fullsized":"2024-09-21T10:03:53.843912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830367+0000","last_change":"2024-09-21T10:02:46.662651+0000","last_active":"2024-09-21T10:03:53.830367+0000","last_peered":"2024-09-21T10:03:53.830367+0000","last_clean":"2024-09-21T10:03:53.830367+0000","last_became_active":"2024-09-21T10:02:46.662304+0000","last_became_peered":"2024-09-21T10:02:46.662304+0000","last_unstale":"2024-09-21T10:03:53.830367+0000","last_undegraded":"2024-09-21T10:03:53.830367+0000","last_fullsized":"2024-09-21T10:03:53.830367+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:06.746929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840595+0000","last_change":"2024-09-21T10:03:53.840511+0000","last_active":"2024-09-21T10:03:53.840595+0000","last_peered":"2024-09-21T10:03:53.840595+0000","last_clean":"2024-09-21T10:03:53.840595+0000","last_became_active":"2024-09-21T10:03:53.840136+0000","last_became_peered":"2024-09-21T10:03:53.840136+0000","last_unstale":"2024-09-21T10:03:53.840595+0000","last_undegraded":"2024-09-21T10:03:53.840595+0000","last_fullsized":"2024-09-21T10:03:53.840595+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:24.765990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840155+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:03:53.840155+0000","last_peered":"2024-09-21T10:03:53.840155+0000","last_clean":"2024-09-21T10:03:53.840155+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:03:53.840155+0000","last_undegraded":"2024-09-21T10:03:53.840155+0000","last_fullsized":"2024-09-21T10:03:53.840155+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833841+0000","last_change":"2024-09-21T10:02:46.663900+0000","last_active":"2024-09-21T10:03:52.833841+0000","last_peered":"2024-09-21T10:03:52.833841+0000","last_clean":"2024-09-21T10:03:52.833841+0000","last_became_active":"2024-09-21T10:02:46.663594+0000","last_became_peered":"2024-09-21T10:02:46.663594+0000","last_unstale":"2024-09-21T10:03:52.833841+0000","last_undegraded":"2024-09-21T10:03:52.833841+0000","last_fullsized":"2024-09-21T10:03:52.833841+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:24.140061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832354+0000","last_change":"2024-09-21T10:03:19.765623+0000","last_active":"2024-09-21T10:03:53.832354+0000","last_peered":"2024-09-21T10:03:53.832354+0000","last_clean":"2024-09-21T10:03:53.832354+0000","last_became_active":"2024-09-21T10:03:19.765227+0000","last_became_peered":"2024-09-21T10:03:19.765227+0000","last_unstale":"2024-09-21T10:03:53.832354+0000","last_undegraded":"2024-09-21T10:03:53.832354+0000","last_fullsized":"2024-09-21T10:03:53.832354+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:52.227826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.829335+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:03:52.829335+0000","last_peered":"2024-09-21T10:03:52.829335+0000","last_clean":"2024-09-21T10:03:52.829335+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:03:52.829335+0000","last_undegraded":"2024-09-21T10:03:52.829335+0000","last_fullsized":"2024-09-21T10:03:52.829335+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832211+0000","last_change":"2024-09-21T10:02:46.664014+0000","last_active":"2024-09-21T10:03:53.832211+0000","last_peered":"2024-09-21T10:03:53.832211+0000","last_clean":"2024-09-21T10:03:53.832211+0000","last_became_active":"2024-09-21T10:02:46.663640+0000","last_became_peered":"2024-09-21T10:02:46.663640+0000","last_unstale":"2024-09-21T10:03:53.832211+0000","last_undegraded":"2024-09-21T10:03:53.832211+0000","last_fullsized":"2024-09-21T10:03:53.832211+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:03.733462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845713+0000","last_change":"2024-09-21T10:03:53.845620+0000","last_active":"2024-09-21T10:03:53.845713+0000","last_peered":"2024-09-21T10:03:53.845713+0000","last_clean":"2024-09-21T10:03:53.845713+0000","last_became_active":"2024-09-21T10:03:53.845290+0000","last_became_peered":"2024-09-21T10:03:53.845290+0000","last_unstale":"2024-09-21T10:03:53.845713+0000","last_undegraded":"2024-09-21T10:03:53.845713+0000","last_fullsized":"2024-09-21T10:03:53.845713+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:20.076623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.846700+0000","last_change":"2024-09-21T10:03:53.846606+0000","last_active":"2024-09-21T10:03:53.846700+0000","last_peered":"2024-09-21T10:03:53.846700+0000","last_clean":"2024-09-21T10:03:53.846700+0000","last_became_active":"2024-09-21T10:03:53.846245+0000","last_became_peered":"2024-09-21T10:03:53.846245+0000","last_unstale":"2024-09-21T10:03:53.846700+0000","last_undegraded":"2024-09-21T10:03:53.846700+0000","last_fullsized":"2024-09-21T10:03:53.846700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:20.745470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":404,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.832350+0000","last_change":"2024-09-21T10:02:12.840225+0000","last_active":"2024-09-21T10:03:52.832350+0000","last_peered":"2024-09-21T10:03:52.832350+0000","last_clean":"2024-09-21T10:03:52.832350+0000","last_became_active":"2024-09-21T10:02:12.839813+0000","last_became_peered":"2024-09-21T10:02:12.839813+0000","last_unstale":"2024-09-21T10:03:52.832350+0000","last_undegraded":"2024-09-21T10:03:52.832350+0000","last_fullsized":"2024-09-21T10:03:52.832350+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:36.630881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":423,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836626+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:03:52.836626+0000","last_peered":"2024-09-21T10:03:52.836626+0000","last_clean":"2024-09-21T10:03:52.836626+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:03:52.836626+0000","last_undegraded":"2024-09-21T10:03:52.836626+0000","last_fullsized":"2024-09-21T10:03:52.836626+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843425+0000","last_change":"2024-09-21T10:03:53.843344+0000","last_active":"2024-09-21T10:03:53.843425+0000","last_peered":"2024-09-21T10:03:53.843425+0000","last_clean":"2024-09-21T10:03:53.843425+0000","last_became_active":"2024-09-21T10:03:53.842304+0000","last_became_peered":"2024-09-21T10:03:53.842304+0000","last_unstale":"2024-09-21T10:03:53.843425+0000","last_undegraded":"2024-09-21T10:03:53.843425+0000","last_fullsized":"2024-09-21T10:03:53.843425+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:31.221735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833226+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:03:52.833226+0000","last_peered":"2024-09-21T10:03:52.833226+0000","last_clean":"2024-09-21T10:03:52.833226+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:03:52.833226+0000","last_undegraded":"2024-09-21T10:03:52.833226+0000","last_fullsized":"2024-09-21T10:03:52.833226+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":568,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.832252+0000","last_change":"2024-09-21T10:02:46.663394+0000","last_active":"2024-09-21T10:03:52.832252+0000","last_peered":"2024-09-21T10:03:52.832252+0000","last_clean":"2024-09-21T10:03:52.832252+0000","last_became_active":"2024-09-21T10:02:46.662879+0000","last_became_peered":"2024-09-21T10:02:46.662879+0000","last_unstale":"2024-09-21T10:03:52.832252+0000","last_undegraded":"2024-09-21T10:03:52.832252+0000","last_fullsized":"2024-09-21T10:03:52.832252+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.125795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835797+0000","last_change":"2024-09-21T10:03:53.835329+0000","last_active":"2024-09-21T10:03:53.835797+0000","last_peered":"2024-09-21T10:03:53.835797+0000","last_clean":"2024-09-21T10:03:53.835797+0000","last_became_active":"2024-09-21T10:03:53.834934+0000","last_became_peered":"2024-09-21T10:03:53.834934+0000","last_unstale":"2024-09-21T10:03:53.835797+0000","last_undegraded":"2024-09-21T10:03:53.835797+0000","last_fullsized":"2024-09-21T10:03:53.835797+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:38.355817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830913+0000","last_change":"2024-09-21T10:03:19.770016+0000","last_active":"2024-09-21T10:03:53.830913+0000","last_peered":"2024-09-21T10:03:53.830913+0000","last_clean":"2024-09-21T10:03:53.830913+0000","last_became_active":"2024-09-21T10:03:19.768866+0000","last_became_peered":"2024-09-21T10:03:19.768866+0000","last_unstale":"2024-09-21T10:03:53.830913+0000","last_undegraded":"2024-09-21T10:03:53.830913+0000","last_fullsized":"2024-09-21T10:03:53.830913+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:25.068940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":573,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840408+0000","last_change":"2024-09-21T10:03:53.840322+0000","last_active":"2024-09-21T10:03:53.840408+0000","last_peered":"2024-09-21T10:03:53.840408+0000","last_clean":"2024-09-21T10:03:53.840408+0000","last_became_active":"2024-09-21T10:03:53.839753+0000","last_became_peered":"2024-09-21T10:03:53.839753+0000","last_unstale":"2024-09-21T10:03:53.840408+0000","last_undegraded":"2024-09-21T10:03:53.840408+0000","last_fullsized":"2024-09-21T10:03:53.840408+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:26.364888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835436+0000","last_change":"2024-09-21T10:03:19.765402+0000","last_active":"2024-09-21T10:03:53.835436+0000","last_peered":"2024-09-21T10:03:53.835436+0000","last_clean":"2024-09-21T10:03:53.835436+0000","last_became_active":"2024-09-21T10:03:19.765036+0000","last_became_peered":"2024-09-21T10:03:19.765036+0000","last_unstale":"2024-09-21T10:03:53.835436+0000","last_undegraded":"2024-09-21T10:03:53.835436+0000","last_fullsized":"2024-09-21T10:03:53.835436+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:32.230377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837881+0000","last_change":"2024-09-21T10:02:46.660814+0000","last_active":"2024-09-21T10:03:53.837881+0000","last_peered":"2024-09-21T10:03:53.837881+0000","last_clean":"2024-09-21T10:03:53.837881+0000","last_became_active":"2024-09-21T10:02:46.660325+0000","last_became_peered":"2024-09-21T10:02:46.660325+0000","last_unstale":"2024-09-21T10:03:53.837881+0000","last_undegraded":"2024-09-21T10:03:53.837881+0000","last_fullsized":"2024-09-21T10:03:53.837881+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:44.427243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":811,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836665+0000","last_change":"2024-09-21T10:02:46.657107+0000","last_active":"2024-09-21T10:03:52.836665+0000","last_peered":"2024-09-21T10:03:52.836665+0000","last_clean":"2024-09-21T10:03:52.836665+0000","last_became_active":"2024-09-21T10:02:46.656633+0000","last_became_peered":"2024-09-21T10:02:46.656633+0000","last_unstale":"2024-09-21T10:03:52.836665+0000","last_undegraded":"2024-09-21T10:03:52.836665+0000","last_fullsized":"2024-09-21T10:03:52.836665+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:42:18.534832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.830018+0000","last_change":"2024-09-21T10:02:12.840581+0000","last_active":"2024-09-21T10:03:52.830018+0000","last_peered":"2024-09-21T10:03:52.830018+0000","last_clean":"2024-09-21T10:03:52.830018+0000","last_became_active":"2024-09-21T10:02:12.840033+0000","last_became_peered":"2024-09-21T10:02:12.840033+0000","last_unstale":"2024-09-21T10:03:52.830018+0000","last_undegraded":"2024-09-21T10:03:52.830018+0000","last_fullsized":"2024-09-21T10:03:52.830018+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:22.449586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843653+0000","last_change":"2024-09-21T10:03:53.843551+0000","last_active":"2024-09-21T10:03:53.843653+0000","last_peered":"2024-09-21T10:03:53.843653+0000","last_clean":"2024-09-21T10:03:53.843653+0000","last_became_active":"2024-09-21T10:03:53.843007+0000","last_became_peered":"2024-09-21T10:03:53.843007+0000","last_unstale":"2024-09-21T10:03:53.843653+0000","last_undegraded":"2024-09-21T10:03:53.843653+0000","last_fullsized":"2024-09-21T10:03:53.843653+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:11.757447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843567+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:03:53.843567+0000","last_peered":"2024-09-21T10:03:53.843567+0000","last_clean":"2024-09-21T10:03:53.843567+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:03:53.843567+0000","last_undegraded":"2024-09-21T10:03:53.843567+0000","last_fullsized":"2024-09-21T10:03:53.843567+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":529,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.836943+0000","last_change":"2024-09-21T10:02:46.662410+0000","last_active":"2024-09-21T10:03:53.836943+0000","last_peered":"2024-09-21T10:03:53.836943+0000","last_clean":"2024-09-21T10:03:53.836943+0000","last_became_active":"2024-09-21T10:02:46.661506+0000","last_became_peered":"2024-09-21T10:02:46.661506+0000","last_unstale":"2024-09-21T10:03:53.836943+0000","last_undegraded":"2024-09-21T10:03:53.836943+0000","last_fullsized":"2024-09-21T10:03:53.836943+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:40:33.566844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834903+0000","last_change":"2024-09-21T10:02:46.665713+0000","last_active":"2024-09-21T10:03:53.834903+0000","last_peered":"2024-09-21T10:03:53.834903+0000","last_clean":"2024-09-21T10:03:53.834903+0000","last_became_active":"2024-09-21T10:02:46.665316+0000","last_became_peered":"2024-09-21T10:02:46.665316+0000","last_unstale":"2024-09-21T10:03:53.834903+0000","last_undegraded":"2024-09-21T10:03:53.834903+0000","last_fullsized":"2024-09-21T10:03:53.834903+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.951202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837083+0000","last_change":"2024-09-21T10:03:19.768187+0000","last_active":"2024-09-21T10:03:53.837083+0000","last_peered":"2024-09-21T10:03:53.837083+0000","last_clean":"2024-09-21T10:03:53.837083+0000","last_became_active":"2024-09-21T10:03:19.767801+0000","last_became_peered":"2024-09-21T10:03:19.767801+0000","last_unstale":"2024-09-21T10:03:53.837083+0000","last_undegraded":"2024-09-21T10:03:53.837083+0000","last_fullsized":"2024-09-21T10:03:53.837083+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:38.042481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":511,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835831+0000","last_change":"2024-09-21T10:02:12.840296+0000","last_active":"2024-09-21T10:03:53.835831+0000","last_peered":"2024-09-21T10:03:53.835831+0000","last_clean":"2024-09-21T10:03:53.835831+0000","last_became_active":"2024-09-21T10:02:12.839881+0000","last_became_peered":"2024-09-21T10:02:12.839881+0000","last_unstale":"2024-09-21T10:03:53.835831+0000","last_undegraded":"2024-09-21T10:03:53.835831+0000","last_fullsized":"2024-09-21T10:03:53.835831+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:49.166865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830265+0000","last_change":"2024-09-21T10:02:12.845158+0000","last_active":"2024-09-21T10:03:53.830265+0000","last_peered":"2024-09-21T10:03:53.830265+0000","last_clean":"2024-09-21T10:03:53.830265+0000","last_became_active":"2024-09-21T10:02:12.843399+0000","last_became_peered":"2024-09-21T10:02:12.843399+0000","last_unstale":"2024-09-21T10:03:53.830265+0000","last_undegraded":"2024-09-21T10:03:53.830265+0000","last_fullsized":"2024-09-21T10:03:53.830265+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:27.552491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842700+0000","last_change":"2024-09-21T10:03:53.842610+0000","last_active":"2024-09-21T10:03:53.842700+0000","last_peered":"2024-09-21T10:03:53.842700+0000","last_clean":"2024-09-21T10:03:53.842700+0000","last_became_active":"2024-09-21T10:03:53.841888+0000","last_became_peered":"2024-09-21T10:03:53.841888+0000","last_unstale":"2024-09-21T10:03:53.842700+0000","last_undegraded":"2024-09-21T10:03:53.842700+0000","last_fullsized":"2024-09-21T10:03:53.842700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:05.833771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":635,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835262+0000","last_change":"2024-09-21T10:02:46.665711+0000","last_active":"2024-09-21T10:03:53.835262+0000","last_peered":"2024-09-21T10:03:53.835262+0000","last_clean":"2024-09-21T10:03:53.835262+0000","last_became_active":"2024-09-21T10:02:46.665165+0000","last_became_peered":"2024-09-21T10:02:46.665165+0000","last_unstale":"2024-09-21T10:03:53.835262+0000","last_undegraded":"2024-09-21T10:03:53.835262+0000","last_fullsized":"2024-09-21T10:03:53.835262+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:08.886554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839031+0000","last_change":"2024-09-21T10:03:53.838941+0000","last_active":"2024-09-21T10:03:53.839031+0000","last_peered":"2024-09-21T10:03:53.839031+0000","last_clean":"2024-09-21T10:03:53.839031+0000","last_became_active":"2024-09-21T10:03:53.838646+0000","last_became_peered":"2024-09-21T10:03:53.838646+0000","last_unstale":"2024-09-21T10:03:53.839031+0000","last_undegraded":"2024-09-21T10:03:53.839031+0000","last_fullsized":"2024-09-21T10:03:53.839031+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:25.167057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830074+0000","last_change":"2024-09-21T10:02:46.661363+0000","last_active":"2024-09-21T10:03:53.830074+0000","last_peered":"2024-09-21T10:03:53.830074+0000","last_clean":"2024-09-21T10:03:53.830074+0000","last_became_active":"2024-09-21T10:02:46.661005+0000","last_became_peered":"2024-09-21T10:02:46.661005+0000","last_unstale":"2024-09-21T10:03:53.830074+0000","last_undegraded":"2024-09-21T10:03:53.830074+0000","last_fullsized":"2024-09-21T10:03:53.830074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:42.908471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":349,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.838167+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:03:53.838167+0000","last_peered":"2024-09-21T10:03:53.838167+0000","last_clean":"2024-09-21T10:03:53.838167+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:03:53.838167+0000","last_undegraded":"2024-09-21T10:03:53.838167+0000","last_fullsized":"2024-09-21T10:03:53.838167+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.829108+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:03:52.829108+0000","last_peered":"2024-09-21T10:03:52.829108+0000","last_clean":"2024-09-21T10:03:52.829108+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:03:52.829108+0000","last_undegraded":"2024-09-21T10:03:52.829108+0000","last_fullsized":"2024-09-21T10:03:52.829108+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832815+0000","last_change":"2024-09-21T10:02:46.667831+0000","last_active":"2024-09-21T10:03:53.832815+0000","last_peered":"2024-09-21T10:03:53.832815+0000","last_clean":"2024-09-21T10:03:53.832815+0000","last_became_active":"2024-09-21T10:02:46.667507+0000","last_became_peered":"2024-09-21T10:02:46.667507+0000","last_unstale":"2024-09-21T10:03:53.832815+0000","last_undegraded":"2024-09-21T10:03:53.832815+0000","last_fullsized":"2024-09-21T10:03:53.832815+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:23.990802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832211+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:03:53.832211+0000","last_peered":"2024-09-21T10:03:53.832211+0000","last_clean":"2024-09-21T10:03:53.832211+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:03:53.832211+0000","last_undegraded":"2024-09-21T10:03:53.832211+0000","last_fullsized":"2024-09-21T10:03:53.832211+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836450+0000","last_change":"2024-09-21T10:03:19.767147+0000","last_active":"2024-09-21T10:03:52.836450+0000","last_peered":"2024-09-21T10:03:52.836450+0000","last_clean":"2024-09-21T10:03:52.836450+0000","last_became_active":"2024-09-21T10:03:19.766822+0000","last_became_peered":"2024-09-21T10:03:19.766822+0000","last_unstale":"2024-09-21T10:03:52.836450+0000","last_undegraded":"2024-09-21T10:03:52.836450+0000","last_fullsized":"2024-09-21T10:03:52.836450+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:38.875912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.841826+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:03:53.841826+0000","last_peered":"2024-09-21T10:03:53.841826+0000","last_clean":"2024-09-21T10:03:53.841826+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:03:53.841826+0000","last_undegraded":"2024-09-21T10:03:53.841826+0000","last_fullsized":"2024-09-21T10:03:53.841826+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836449+0000","last_change":"2024-09-21T10:02:46.659383+0000","last_active":"2024-09-21T10:03:52.836449+0000","last_peered":"2024-09-21T10:03:52.836449+0000","last_clean":"2024-09-21T10:03:52.836449+0000","last_became_active":"2024-09-21T10:02:46.659011+0000","last_became_peered":"2024-09-21T10:02:46.659011+0000","last_unstale":"2024-09-21T10:03:52.836449+0000","last_undegraded":"2024-09-21T10:03:52.836449+0000","last_fullsized":"2024-09-21T10:03:52.836449+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:04.650699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845276+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:03:53.845276+0000","last_peered":"2024-09-21T10:03:53.845276+0000","last_clean":"2024-09-21T10:03:53.845276+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:03:53.845276+0000","last_undegraded":"2024-09-21T10:03:53.845276+0000","last_fullsized":"2024-09-21T10:03:53.845276+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.844990+0000","last_change":"2024-09-21T10:03:53.844673+0000","last_active":"2024-09-21T10:03:53.844990+0000","last_peered":"2024-09-21T10:03:53.844990+0000","last_clean":"2024-09-21T10:03:53.844990+0000","last_became_active":"2024-09-21T10:03:53.844313+0000","last_became_peered":"2024-09-21T10:03:53.844313+0000","last_unstale":"2024-09-21T10:03:53.844990+0000","last_undegraded":"2024-09-21T10:03:53.844990+0000","last_fullsized":"2024-09-21T10:03:53.844990+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:42.529301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.841503+0000","last_change":"2024-09-21T10:03:53.841417+0000","last_active":"2024-09-21T10:03:53.841503+0000","last_peered":"2024-09-21T10:03:53.841503+0000","last_clean":"2024-09-21T10:03:53.841503+0000","last_became_active":"2024-09-21T10:03:53.840993+0000","last_became_peered":"2024-09-21T10:03:53.840993+0000","last_unstale":"2024-09-21T10:03:53.841503+0000","last_undegraded":"2024-09-21T10:03:53.841503+0000","last_fullsized":"2024-09-21T10:03:53.841503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:05.666490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":81,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.834009+0000","last_change":"2024-09-21T10:03:19.765682+0000","last_active":"2024-09-21T10:03:52.834009+0000","last_peered":"2024-09-21T10:03:52.834009+0000","last_clean":"2024-09-21T10:03:52.834009+0000","last_became_active":"2024-09-21T10:03:19.765344+0000","last_became_peered":"2024-09-21T10:03:19.765344+0000","last_unstale":"2024-09-21T10:03:52.834009+0000","last_undegraded":"2024-09-21T10:03:52.834009+0000","last_fullsized":"2024-09-21T10:03:52.834009+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:43.787280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839452+0000","last_change":"2024-09-21T10:03:53.839372+0000","last_active":"2024-09-21T10:03:53.839452+0000","last_peered":"2024-09-21T10:03:53.839452+0000","last_clean":"2024-09-21T10:03:53.839452+0000","last_became_active":"2024-09-21T10:03:53.839053+0000","last_became_peered":"2024-09-21T10:03:53.839053+0000","last_unstale":"2024-09-21T10:03:53.839452+0000","last_undegraded":"2024-09-21T10:03:53.839452+0000","last_fullsized":"2024-09-21T10:03:53.839452+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:29.647982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845364+0000","last_change":"2024-09-21T10:03:53.844946+0000","last_active":"2024-09-21T10:03:53.845364+0000","last_peered":"2024-09-21T10:03:53.845364+0000","last_clean":"2024-09-21T10:03:53.845364+0000","last_became_active":"2024-09-21T10:03:53.844613+0000","last_became_peered":"2024-09-21T10:03:53.844613+0000","last_unstale":"2024-09-21T10:03:53.845364+0000","last_undegraded":"2024-09-21T10:03:53.845364+0000","last_fullsized":"2024-09-21T10:03:53.845364+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:41.055573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.847034+0000","last_change":"2024-09-21T10:03:53.846948+0000","last_active":"2024-09-21T10:03:53.847034+0000","last_peered":"2024-09-21T10:03:53.847034+0000","last_clean":"2024-09-21T10:03:53.847034+0000","last_became_active":"2024-09-21T10:03:53.846614+0000","last_became_peered":"2024-09-21T10:03:53.846614+0000","last_unstale":"2024-09-21T10:03:53.847034+0000","last_undegraded":"2024-09-21T10:03:53.847034+0000","last_fullsized":"2024-09-21T10:03:53.847034+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:53.968372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845285+0000","last_change":"2024-09-21T10:03:53.845201+0000","last_active":"2024-09-21T10:03:53.845285+0000","last_peered":"2024-09-21T10:03:53.845285+0000","last_clean":"2024-09-21T10:03:53.845285+0000","last_became_active":"2024-09-21T10:03:53.843822+0000","last_became_peered":"2024-09-21T10:03:53.843822+0000","last_unstale":"2024-09-21T10:03:53.845285+0000","last_undegraded":"2024-09-21T10:03:53.845285+0000","last_fullsized":"2024-09-21T10:03:53.845285+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:28.895373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.828363+0000","last_change":"2024-09-21T10:03:19.768025+0000","last_active":"2024-09-21T10:03:52.828363+0000","last_peered":"2024-09-21T10:03:52.828363+0000","last_clean":"2024-09-21T10:03:52.828363+0000","last_became_active":"2024-09-21T10:03:19.767632+0000","last_became_peered":"2024-09-21T10:03:19.767632+0000","last_unstale":"2024-09-21T10:03:52.828363+0000","last_undegraded":"2024-09-21T10:03:52.828363+0000","last_fullsized":"2024-09-21T10:03:52.828363+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:40.023759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840358+0000","last_change":"2024-09-21T10:03:53.840263+0000","last_active":"2024-09-21T10:03:53.840358+0000","last_peered":"2024-09-21T10:03:53.840358+0000","last_clean":"2024-09-21T10:03:53.840358+0000","last_became_active":"2024-09-21T10:03:53.839894+0000","last_became_peered":"2024-09-21T10:03:53.839894+0000","last_unstale":"2024-09-21T10:03:53.840358+0000","last_undegraded":"2024-09-21T10:03:53.840358+0000","last_fullsized":"2024-09-21T10:03:53.840358+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:52.489580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830275+0000","last_change":"2024-09-21T10:02:12.846054+0000","last_active":"2024-09-21T10:03:53.830275+0000","last_peered":"2024-09-21T10:03:53.830275+0000","last_clean":"2024-09-21T10:03:53.830275+0000","last_became_active":"2024-09-21T10:02:12.843696+0000","last_became_peered":"2024-09-21T10:02:12.843696+0000","last_unstale":"2024-09-21T10:03:53.830275+0000","last_undegraded":"2024-09-21T10:03:53.830275+0000","last_fullsized":"2024-09-21T10:03:53.830275+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:32.280697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840992+0000","last_change":"2024-09-21T10:03:53.840992+0000","last_active":"2024-09-21T10:03:53.840992+0000","last_peered":"2024-09-21T10:03:53.840992+0000","last_clean":"2024-09-21T10:03:53.840992+0000","last_became_active":"2024-09-21T10:03:53.840551+0000","last_became_peered":"2024-09-21T10:03:53.840551+0000","last_unstale":"2024-09-21T10:03:53.840992+0000","last_undegraded":"2024-09-21T10:03:53.840992+0000","last_fullsized":"2024-09-21T10:03:53.840992+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:04.442126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836717+0000","last_change":"2024-09-21T10:02:46.659513+0000","last_active":"2024-09-21T10:03:52.836717+0000","last_peered":"2024-09-21T10:03:52.836717+0000","last_clean":"2024-09-21T10:03:52.836717+0000","last_became_active":"2024-09-21T10:02:46.659130+0000","last_became_peered":"2024-09-21T10:02:46.659130+0000","last_unstale":"2024-09-21T10:03:52.836717+0000","last_undegraded":"2024-09-21T10:03:52.836717+0000","last_fullsized":"2024-09-21T10:03:52.836717+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:24.792175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835006+0000","last_change":"2024-09-21T10:03:53.835006+0000","last_active":"2024-09-21T10:03:53.835006+0000","last_peered":"2024-09-21T10:03:53.835006+0000","last_clean":"2024-09-21T10:03:53.835006+0000","last_became_active":"2024-09-21T10:03:53.834608+0000","last_became_peered":"2024-09-21T10:03:53.834608+0000","last_unstale":"2024-09-21T10:03:53.835006+0000","last_undegraded":"2024-09-21T10:03:53.835006+0000","last_fullsized":"2024-09-21T10:03:53.835006+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:56.108606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":59,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.837866+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:03:52.837866+0000","last_peered":"2024-09-21T10:03:52.837866+0000","last_clean":"2024-09-21T10:03:52.837866+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:03:52.837866+0000","last_undegraded":"2024-09-21T10:03:52.837866+0000","last_fullsized":"2024-09-21T10:03:52.837866+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.844128+0000","last_change":"2024-09-21T10:03:53.844043+0000","last_active":"2024-09-21T10:03:53.844128+0000","last_peered":"2024-09-21T10:03:53.844128+0000","last_clean":"2024-09-21T10:03:53.844128+0000","last_became_active":"2024-09-21T10:03:53.842891+0000","last_became_peered":"2024-09-21T10:03:53.842891+0000","last_unstale":"2024-09-21T10:03:53.844128+0000","last_undegraded":"2024-09-21T10:03:53.844128+0000","last_fullsized":"2024-09-21T10:03:53.844128+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:39.891977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842964+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:03:53.842964+0000","last_peered":"2024-09-21T10:03:53.842964+0000","last_clean":"2024-09-21T10:03:53.842964+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:03:53.842964+0000","last_undegraded":"2024-09-21T10:03:53.842964+0000","last_fullsized":"2024-09-21T10:03:53.842964+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.828174+0000","last_change":"2024-09-21T10:02:46.659818+0000","last_active":"2024-09-21T10:03:52.828174+0000","last_peered":"2024-09-21T10:03:52.828174+0000","last_clean":"2024-09-21T10:03:52.828174+0000","last_became_active":"2024-09-21T10:02:46.659457+0000","last_became_peered":"2024-09-21T10:02:46.659457+0000","last_unstale":"2024-09-21T10:03:52.828174+0000","last_undegraded":"2024-09-21T10:03:52.828174+0000","last_fullsized":"2024-09-21T10:03:52.828174+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:21.962330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830391+0000","last_change":"2024-09-21T10:03:19.769549+0000","last_active":"2024-09-21T10:03:53.830391+0000","last_peered":"2024-09-21T10:03:53.830391+0000","last_clean":"2024-09-21T10:03:53.830391+0000","last_became_active":"2024-09-21T10:03:19.768854+0000","last_became_peered":"2024-09-21T10:03:19.768854+0000","last_unstale":"2024-09-21T10:03:53.830391+0000","last_undegraded":"2024-09-21T10:03:53.830391+0000","last_fullsized":"2024-09-21T10:03:53.830391+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:29.415039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234181,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37008,"kb_used_data":5884,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708144,"statfs":{"total":95995035648,"available":95957139456,"internally_reserved":0,"allocated":6025216,"data_stored":5417204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495819,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32908,"kb_used_data":1784,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712244,"statfs":{"total":95995035648,"available":95961337856,"internally_reserved":0,"allocated":1826816,"data_stored":1223104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757456,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28216,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716936,"statfs":{"total":95995035648,"available":95966142464,"internally_reserved":0,"allocated":1224704,"data_stored":628562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019092,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32332,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712820,"statfs":{"total":95995035648,"available":95961927680,"internally_reserved":0,"allocated":1236992,"data_stored":632793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435276,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32888,"kb_used_data":1188,"kb_used_omap":5,"kb_used_meta":31674,"kb_avail":93712264,"statfs":{"total":95995035648,"available":95961358336,"internally_reserved":0,"allocated":1216512,"data_stored":627993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":32434216},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598803,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1188,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1216512,"data_stored":628211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":28172749},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762330,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36976,"kb_used_data":5280,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708176,"statfs":{"total":95995035648,"available":95957172224,"internally_reserved":0,"allocated":5406720,"data_stored":4821953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":32432381},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925857,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33404,"kb_used_data":5868,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711748,"statfs":{"total":95995035648,"available":95960829952,"internally_reserved":0,"allocated":6008832,"data_stored":5413179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":28172478},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:04:47.596 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:04:47.811 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:47.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:47 smithi032 ceph-mon[101299]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:47.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:47 smithi032 ceph-mon[105748]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:48.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:47 smithi045 ceph-mon[95976]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:48.935 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:48 smithi032 ceph-mon[105748]: from='client.34539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:48.936 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:48.936 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:48 smithi032 ceph-mon[101299]: from='client.34539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:48.936 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:48 smithi045 ceph-mon[95976]: from='client.34539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:49.659 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:04:49.660 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:04:49.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:49 smithi032 ceph-mon[101299]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:49 smithi032 ceph-mon[105748]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:04:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:04:49] "GET /metrics HTTP/1.1" 200 33920 "" "Prometheus/2.51.0" 2024-09-21T10:04:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:49 smithi045 ceph-mon[95976]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:50.379 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":278,"stamp":"2024-09-21T10:04:48.122748+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":262452,"kb_used_data":23596,"kb_used_omap":35,"kb_used_meta":238684,"kb_avail":749698764,"statfs":{"total":767960285184,"available":767691534336,"internally_reserved":0,"allocated":24162304,"data_stored":19392999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":244413134},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002733"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843669+0000","last_change":"2024-09-21T10:03:53.843548+0000","last_active":"2024-09-21T10:03:53.843669+0000","last_peered":"2024-09-21T10:03:53.843669+0000","last_clean":"2024-09-21T10:03:53.843669+0000","last_became_active":"2024-09-21T10:03:53.843182+0000","last_became_peered":"2024-09-21T10:03:53.843182+0000","last_unstale":"2024-09-21T10:03:53.843669+0000","last_undegraded":"2024-09-21T10:03:53.843669+0000","last_fullsized":"2024-09-21T10:03:53.843669+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:09.464209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834897+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:03:53.834897+0000","last_peered":"2024-09-21T10:03:53.834897+0000","last_clean":"2024-09-21T10:03:53.834897+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:03:53.834897+0000","last_undegraded":"2024-09-21T10:03:53.834897+0000","last_fullsized":"2024-09-21T10:03:53.834897+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839987+0000","last_change":"2024-09-21T10:03:53.839987+0000","last_active":"2024-09-21T10:03:53.839987+0000","last_peered":"2024-09-21T10:03:53.839987+0000","last_clean":"2024-09-21T10:03:53.839987+0000","last_became_active":"2024-09-21T10:03:53.839664+0000","last_became_peered":"2024-09-21T10:03:53.839664+0000","last_unstale":"2024-09-21T10:03:53.839987+0000","last_undegraded":"2024-09-21T10:03:53.839987+0000","last_fullsized":"2024-09-21T10:03:53.839987+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:51.726392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837862+0000","last_change":"2024-09-21T10:03:53.837496+0000","last_active":"2024-09-21T10:03:53.837862+0000","last_peered":"2024-09-21T10:03:53.837862+0000","last_clean":"2024-09-21T10:03:53.837862+0000","last_became_active":"2024-09-21T10:03:53.837166+0000","last_became_peered":"2024-09-21T10:03:53.837166+0000","last_unstale":"2024-09-21T10:03:53.837862+0000","last_undegraded":"2024-09-21T10:03:53.837862+0000","last_fullsized":"2024-09-21T10:03:53.837862+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:45.795571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845452+0000","last_change":"2024-09-21T10:03:53.844990+0000","last_active":"2024-09-21T10:03:53.845452+0000","last_peered":"2024-09-21T10:03:53.845452+0000","last_clean":"2024-09-21T10:03:53.845452+0000","last_became_active":"2024-09-21T10:03:53.844664+0000","last_became_peered":"2024-09-21T10:03:53.844664+0000","last_unstale":"2024-09-21T10:03:53.845452+0000","last_undegraded":"2024-09-21T10:03:53.845452+0000","last_fullsized":"2024-09-21T10:03:53.845452+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:01.031522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835247+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:03:53.835247+0000","last_peered":"2024-09-21T10:03:53.835247+0000","last_clean":"2024-09-21T10:03:53.835247+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:03:53.835247+0000","last_undegraded":"2024-09-21T10:03:53.835247+0000","last_fullsized":"2024-09-21T10:03:53.835247+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845872+0000","last_change":"2024-09-21T10:03:53.845785+0000","last_active":"2024-09-21T10:03:53.845872+0000","last_peered":"2024-09-21T10:03:53.845872+0000","last_clean":"2024-09-21T10:03:53.845872+0000","last_became_active":"2024-09-21T10:03:53.845303+0000","last_became_peered":"2024-09-21T10:03:53.845303+0000","last_unstale":"2024-09-21T10:03:53.845872+0000","last_undegraded":"2024-09-21T10:03:53.845872+0000","last_fullsized":"2024-09-21T10:03:53.845872+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:04.976007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837943+0000","last_change":"2024-09-21T10:02:12.838786+0000","last_active":"2024-09-21T10:03:53.837943+0000","last_peered":"2024-09-21T10:03:53.837943+0000","last_clean":"2024-09-21T10:03:53.837943+0000","last_became_active":"2024-09-21T10:02:12.838242+0000","last_became_peered":"2024-09-21T10:02:12.838242+0000","last_unstale":"2024-09-21T10:03:53.837943+0000","last_undegraded":"2024-09-21T10:03:53.837943+0000","last_fullsized":"2024-09-21T10:03:53.837943+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:31.071136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835170+0000","last_change":"2024-09-21T10:02:12.840334+0000","last_active":"2024-09-21T10:03:53.835170+0000","last_peered":"2024-09-21T10:03:53.835170+0000","last_clean":"2024-09-21T10:03:53.835170+0000","last_became_active":"2024-09-21T10:02:12.839928+0000","last_became_peered":"2024-09-21T10:02:12.839928+0000","last_unstale":"2024-09-21T10:03:53.835170+0000","last_undegraded":"2024-09-21T10:03:53.835170+0000","last_fullsized":"2024-09-21T10:03:53.835170+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:38.593644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836136+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T10:03:52.836136+0000","last_peered":"2024-09-21T10:03:52.836136+0000","last_clean":"2024-09-21T10:03:52.836136+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T10:03:52.836136+0000","last_undegraded":"2024-09-21T10:03:52.836136+0000","last_fullsized":"2024-09-21T10:03:52.836136+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.836116+0000","last_change":"2024-09-21T10:02:46.662169+0000","last_active":"2024-09-21T10:03:53.836116+0000","last_peered":"2024-09-21T10:03:53.836116+0000","last_clean":"2024-09-21T10:03:53.836116+0000","last_became_active":"2024-09-21T10:02:46.661721+0000","last_became_peered":"2024-09-21T10:02:46.661721+0000","last_unstale":"2024-09-21T10:03:53.836116+0000","last_undegraded":"2024-09-21T10:03:53.836116+0000","last_fullsized":"2024-09-21T10:03:53.836116+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:10.414152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840738+0000","last_change":"2024-09-21T10:03:53.840738+0000","last_active":"2024-09-21T10:03:53.840738+0000","last_peered":"2024-09-21T10:03:53.840738+0000","last_clean":"2024-09-21T10:03:53.840738+0000","last_became_active":"2024-09-21T10:03:53.840363+0000","last_became_peered":"2024-09-21T10:03:53.840363+0000","last_unstale":"2024-09-21T10:03:53.840738+0000","last_undegraded":"2024-09-21T10:03:53.840738+0000","last_fullsized":"2024-09-21T10:03:53.840738+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:26.371181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835739+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T10:03:53.835739+0000","last_peered":"2024-09-21T10:03:53.835739+0000","last_clean":"2024-09-21T10:03:53.835739+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T10:03:53.835739+0000","last_undegraded":"2024-09-21T10:03:53.835739+0000","last_fullsized":"2024-09-21T10:03:53.835739+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833635+0000","last_change":"2024-09-21T10:02:46.663127+0000","last_active":"2024-09-21T10:03:52.833635+0000","last_peered":"2024-09-21T10:03:52.833635+0000","last_clean":"2024-09-21T10:03:52.833635+0000","last_became_active":"2024-09-21T10:02:46.662791+0000","last_became_peered":"2024-09-21T10:02:46.662791+0000","last_unstale":"2024-09-21T10:03:52.833635+0000","last_undegraded":"2024-09-21T10:03:52.833635+0000","last_fullsized":"2024-09-21T10:03:52.833635+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:59.990122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843912+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:03:53.843912+0000","last_peered":"2024-09-21T10:03:53.843912+0000","last_clean":"2024-09-21T10:03:53.843912+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:03:53.843912+0000","last_undegraded":"2024-09-21T10:03:53.843912+0000","last_fullsized":"2024-09-21T10:03:53.843912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830367+0000","last_change":"2024-09-21T10:02:46.662651+0000","last_active":"2024-09-21T10:03:53.830367+0000","last_peered":"2024-09-21T10:03:53.830367+0000","last_clean":"2024-09-21T10:03:53.830367+0000","last_became_active":"2024-09-21T10:02:46.662304+0000","last_became_peered":"2024-09-21T10:02:46.662304+0000","last_unstale":"2024-09-21T10:03:53.830367+0000","last_undegraded":"2024-09-21T10:03:53.830367+0000","last_fullsized":"2024-09-21T10:03:53.830367+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:06.746929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840595+0000","last_change":"2024-09-21T10:03:53.840511+0000","last_active":"2024-09-21T10:03:53.840595+0000","last_peered":"2024-09-21T10:03:53.840595+0000","last_clean":"2024-09-21T10:03:53.840595+0000","last_became_active":"2024-09-21T10:03:53.840136+0000","last_became_peered":"2024-09-21T10:03:53.840136+0000","last_unstale":"2024-09-21T10:03:53.840595+0000","last_undegraded":"2024-09-21T10:03:53.840595+0000","last_fullsized":"2024-09-21T10:03:53.840595+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:24.765990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840155+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:03:53.840155+0000","last_peered":"2024-09-21T10:03:53.840155+0000","last_clean":"2024-09-21T10:03:53.840155+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:03:53.840155+0000","last_undegraded":"2024-09-21T10:03:53.840155+0000","last_fullsized":"2024-09-21T10:03:53.840155+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833841+0000","last_change":"2024-09-21T10:02:46.663900+0000","last_active":"2024-09-21T10:03:52.833841+0000","last_peered":"2024-09-21T10:03:52.833841+0000","last_clean":"2024-09-21T10:03:52.833841+0000","last_became_active":"2024-09-21T10:02:46.663594+0000","last_became_peered":"2024-09-21T10:02:46.663594+0000","last_unstale":"2024-09-21T10:03:52.833841+0000","last_undegraded":"2024-09-21T10:03:52.833841+0000","last_fullsized":"2024-09-21T10:03:52.833841+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:24.140061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832354+0000","last_change":"2024-09-21T10:03:19.765623+0000","last_active":"2024-09-21T10:03:53.832354+0000","last_peered":"2024-09-21T10:03:53.832354+0000","last_clean":"2024-09-21T10:03:53.832354+0000","last_became_active":"2024-09-21T10:03:19.765227+0000","last_became_peered":"2024-09-21T10:03:19.765227+0000","last_unstale":"2024-09-21T10:03:53.832354+0000","last_undegraded":"2024-09-21T10:03:53.832354+0000","last_fullsized":"2024-09-21T10:03:53.832354+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:52.227826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.829335+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:03:52.829335+0000","last_peered":"2024-09-21T10:03:52.829335+0000","last_clean":"2024-09-21T10:03:52.829335+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:03:52.829335+0000","last_undegraded":"2024-09-21T10:03:52.829335+0000","last_fullsized":"2024-09-21T10:03:52.829335+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832211+0000","last_change":"2024-09-21T10:02:46.664014+0000","last_active":"2024-09-21T10:03:53.832211+0000","last_peered":"2024-09-21T10:03:53.832211+0000","last_clean":"2024-09-21T10:03:53.832211+0000","last_became_active":"2024-09-21T10:02:46.663640+0000","last_became_peered":"2024-09-21T10:02:46.663640+0000","last_unstale":"2024-09-21T10:03:53.832211+0000","last_undegraded":"2024-09-21T10:03:53.832211+0000","last_fullsized":"2024-09-21T10:03:53.832211+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:03.733462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845713+0000","last_change":"2024-09-21T10:03:53.845620+0000","last_active":"2024-09-21T10:03:53.845713+0000","last_peered":"2024-09-21T10:03:53.845713+0000","last_clean":"2024-09-21T10:03:53.845713+0000","last_became_active":"2024-09-21T10:03:53.845290+0000","last_became_peered":"2024-09-21T10:03:53.845290+0000","last_unstale":"2024-09-21T10:03:53.845713+0000","last_undegraded":"2024-09-21T10:03:53.845713+0000","last_fullsized":"2024-09-21T10:03:53.845713+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:20.076623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.846700+0000","last_change":"2024-09-21T10:03:53.846606+0000","last_active":"2024-09-21T10:03:53.846700+0000","last_peered":"2024-09-21T10:03:53.846700+0000","last_clean":"2024-09-21T10:03:53.846700+0000","last_became_active":"2024-09-21T10:03:53.846245+0000","last_became_peered":"2024-09-21T10:03:53.846245+0000","last_unstale":"2024-09-21T10:03:53.846700+0000","last_undegraded":"2024-09-21T10:03:53.846700+0000","last_fullsized":"2024-09-21T10:03:53.846700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:20.745470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":404,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.832350+0000","last_change":"2024-09-21T10:02:12.840225+0000","last_active":"2024-09-21T10:03:52.832350+0000","last_peered":"2024-09-21T10:03:52.832350+0000","last_clean":"2024-09-21T10:03:52.832350+0000","last_became_active":"2024-09-21T10:02:12.839813+0000","last_became_peered":"2024-09-21T10:02:12.839813+0000","last_unstale":"2024-09-21T10:03:52.832350+0000","last_undegraded":"2024-09-21T10:03:52.832350+0000","last_fullsized":"2024-09-21T10:03:52.832350+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:36.630881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":423,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836626+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:03:52.836626+0000","last_peered":"2024-09-21T10:03:52.836626+0000","last_clean":"2024-09-21T10:03:52.836626+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:03:52.836626+0000","last_undegraded":"2024-09-21T10:03:52.836626+0000","last_fullsized":"2024-09-21T10:03:52.836626+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843425+0000","last_change":"2024-09-21T10:03:53.843344+0000","last_active":"2024-09-21T10:03:53.843425+0000","last_peered":"2024-09-21T10:03:53.843425+0000","last_clean":"2024-09-21T10:03:53.843425+0000","last_became_active":"2024-09-21T10:03:53.842304+0000","last_became_peered":"2024-09-21T10:03:53.842304+0000","last_unstale":"2024-09-21T10:03:53.843425+0000","last_undegraded":"2024-09-21T10:03:53.843425+0000","last_fullsized":"2024-09-21T10:03:53.843425+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:31.221735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833226+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:03:52.833226+0000","last_peered":"2024-09-21T10:03:52.833226+0000","last_clean":"2024-09-21T10:03:52.833226+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:03:52.833226+0000","last_undegraded":"2024-09-21T10:03:52.833226+0000","last_fullsized":"2024-09-21T10:03:52.833226+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":568,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.832252+0000","last_change":"2024-09-21T10:02:46.663394+0000","last_active":"2024-09-21T10:03:52.832252+0000","last_peered":"2024-09-21T10:03:52.832252+0000","last_clean":"2024-09-21T10:03:52.832252+0000","last_became_active":"2024-09-21T10:02:46.662879+0000","last_became_peered":"2024-09-21T10:02:46.662879+0000","last_unstale":"2024-09-21T10:03:52.832252+0000","last_undegraded":"2024-09-21T10:03:52.832252+0000","last_fullsized":"2024-09-21T10:03:52.832252+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.125795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835797+0000","last_change":"2024-09-21T10:03:53.835329+0000","last_active":"2024-09-21T10:03:53.835797+0000","last_peered":"2024-09-21T10:03:53.835797+0000","last_clean":"2024-09-21T10:03:53.835797+0000","last_became_active":"2024-09-21T10:03:53.834934+0000","last_became_peered":"2024-09-21T10:03:53.834934+0000","last_unstale":"2024-09-21T10:03:53.835797+0000","last_undegraded":"2024-09-21T10:03:53.835797+0000","last_fullsized":"2024-09-21T10:03:53.835797+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:38.355817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830913+0000","last_change":"2024-09-21T10:03:19.770016+0000","last_active":"2024-09-21T10:03:53.830913+0000","last_peered":"2024-09-21T10:03:53.830913+0000","last_clean":"2024-09-21T10:03:53.830913+0000","last_became_active":"2024-09-21T10:03:19.768866+0000","last_became_peered":"2024-09-21T10:03:19.768866+0000","last_unstale":"2024-09-21T10:03:53.830913+0000","last_undegraded":"2024-09-21T10:03:53.830913+0000","last_fullsized":"2024-09-21T10:03:53.830913+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:25.068940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":573,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840408+0000","last_change":"2024-09-21T10:03:53.840322+0000","last_active":"2024-09-21T10:03:53.840408+0000","last_peered":"2024-09-21T10:03:53.840408+0000","last_clean":"2024-09-21T10:03:53.840408+0000","last_became_active":"2024-09-21T10:03:53.839753+0000","last_became_peered":"2024-09-21T10:03:53.839753+0000","last_unstale":"2024-09-21T10:03:53.840408+0000","last_undegraded":"2024-09-21T10:03:53.840408+0000","last_fullsized":"2024-09-21T10:03:53.840408+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:26.364888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835436+0000","last_change":"2024-09-21T10:03:19.765402+0000","last_active":"2024-09-21T10:03:53.835436+0000","last_peered":"2024-09-21T10:03:53.835436+0000","last_clean":"2024-09-21T10:03:53.835436+0000","last_became_active":"2024-09-21T10:03:19.765036+0000","last_became_peered":"2024-09-21T10:03:19.765036+0000","last_unstale":"2024-09-21T10:03:53.835436+0000","last_undegraded":"2024-09-21T10:03:53.835436+0000","last_fullsized":"2024-09-21T10:03:53.835436+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:32.230377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837881+0000","last_change":"2024-09-21T10:02:46.660814+0000","last_active":"2024-09-21T10:03:53.837881+0000","last_peered":"2024-09-21T10:03:53.837881+0000","last_clean":"2024-09-21T10:03:53.837881+0000","last_became_active":"2024-09-21T10:02:46.660325+0000","last_became_peered":"2024-09-21T10:02:46.660325+0000","last_unstale":"2024-09-21T10:03:53.837881+0000","last_undegraded":"2024-09-21T10:03:53.837881+0000","last_fullsized":"2024-09-21T10:03:53.837881+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:44.427243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":811,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836665+0000","last_change":"2024-09-21T10:02:46.657107+0000","last_active":"2024-09-21T10:03:52.836665+0000","last_peered":"2024-09-21T10:03:52.836665+0000","last_clean":"2024-09-21T10:03:52.836665+0000","last_became_active":"2024-09-21T10:02:46.656633+0000","last_became_peered":"2024-09-21T10:02:46.656633+0000","last_unstale":"2024-09-21T10:03:52.836665+0000","last_undegraded":"2024-09-21T10:03:52.836665+0000","last_fullsized":"2024-09-21T10:03:52.836665+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:42:18.534832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.830018+0000","last_change":"2024-09-21T10:02:12.840581+0000","last_active":"2024-09-21T10:03:52.830018+0000","last_peered":"2024-09-21T10:03:52.830018+0000","last_clean":"2024-09-21T10:03:52.830018+0000","last_became_active":"2024-09-21T10:02:12.840033+0000","last_became_peered":"2024-09-21T10:02:12.840033+0000","last_unstale":"2024-09-21T10:03:52.830018+0000","last_undegraded":"2024-09-21T10:03:52.830018+0000","last_fullsized":"2024-09-21T10:03:52.830018+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:22.449586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843653+0000","last_change":"2024-09-21T10:03:53.843551+0000","last_active":"2024-09-21T10:03:53.843653+0000","last_peered":"2024-09-21T10:03:53.843653+0000","last_clean":"2024-09-21T10:03:53.843653+0000","last_became_active":"2024-09-21T10:03:53.843007+0000","last_became_peered":"2024-09-21T10:03:53.843007+0000","last_unstale":"2024-09-21T10:03:53.843653+0000","last_undegraded":"2024-09-21T10:03:53.843653+0000","last_fullsized":"2024-09-21T10:03:53.843653+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:11.757447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843567+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:03:53.843567+0000","last_peered":"2024-09-21T10:03:53.843567+0000","last_clean":"2024-09-21T10:03:53.843567+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:03:53.843567+0000","last_undegraded":"2024-09-21T10:03:53.843567+0000","last_fullsized":"2024-09-21T10:03:53.843567+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":529,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.836943+0000","last_change":"2024-09-21T10:02:46.662410+0000","last_active":"2024-09-21T10:03:53.836943+0000","last_peered":"2024-09-21T10:03:53.836943+0000","last_clean":"2024-09-21T10:03:53.836943+0000","last_became_active":"2024-09-21T10:02:46.661506+0000","last_became_peered":"2024-09-21T10:02:46.661506+0000","last_unstale":"2024-09-21T10:03:53.836943+0000","last_undegraded":"2024-09-21T10:03:53.836943+0000","last_fullsized":"2024-09-21T10:03:53.836943+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:40:33.566844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834903+0000","last_change":"2024-09-21T10:02:46.665713+0000","last_active":"2024-09-21T10:03:53.834903+0000","last_peered":"2024-09-21T10:03:53.834903+0000","last_clean":"2024-09-21T10:03:53.834903+0000","last_became_active":"2024-09-21T10:02:46.665316+0000","last_became_peered":"2024-09-21T10:02:46.665316+0000","last_unstale":"2024-09-21T10:03:53.834903+0000","last_undegraded":"2024-09-21T10:03:53.834903+0000","last_fullsized":"2024-09-21T10:03:53.834903+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.951202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837083+0000","last_change":"2024-09-21T10:03:19.768187+0000","last_active":"2024-09-21T10:03:53.837083+0000","last_peered":"2024-09-21T10:03:53.837083+0000","last_clean":"2024-09-21T10:03:53.837083+0000","last_became_active":"2024-09-21T10:03:19.767801+0000","last_became_peered":"2024-09-21T10:03:19.767801+0000","last_unstale":"2024-09-21T10:03:53.837083+0000","last_undegraded":"2024-09-21T10:03:53.837083+0000","last_fullsized":"2024-09-21T10:03:53.837083+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:38.042481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":511,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835831+0000","last_change":"2024-09-21T10:02:12.840296+0000","last_active":"2024-09-21T10:03:53.835831+0000","last_peered":"2024-09-21T10:03:53.835831+0000","last_clean":"2024-09-21T10:03:53.835831+0000","last_became_active":"2024-09-21T10:02:12.839881+0000","last_became_peered":"2024-09-21T10:02:12.839881+0000","last_unstale":"2024-09-21T10:03:53.835831+0000","last_undegraded":"2024-09-21T10:03:53.835831+0000","last_fullsized":"2024-09-21T10:03:53.835831+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:49.166865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830265+0000","last_change":"2024-09-21T10:02:12.845158+0000","last_active":"2024-09-21T10:03:53.830265+0000","last_peered":"2024-09-21T10:03:53.830265+0000","last_clean":"2024-09-21T10:03:53.830265+0000","last_became_active":"2024-09-21T10:02:12.843399+0000","last_became_peered":"2024-09-21T10:02:12.843399+0000","last_unstale":"2024-09-21T10:03:53.830265+0000","last_undegraded":"2024-09-21T10:03:53.830265+0000","last_fullsized":"2024-09-21T10:03:53.830265+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:27.552491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842700+0000","last_change":"2024-09-21T10:03:53.842610+0000","last_active":"2024-09-21T10:03:53.842700+0000","last_peered":"2024-09-21T10:03:53.842700+0000","last_clean":"2024-09-21T10:03:53.842700+0000","last_became_active":"2024-09-21T10:03:53.841888+0000","last_became_peered":"2024-09-21T10:03:53.841888+0000","last_unstale":"2024-09-21T10:03:53.842700+0000","last_undegraded":"2024-09-21T10:03:53.842700+0000","last_fullsized":"2024-09-21T10:03:53.842700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:05.833771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":635,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835262+0000","last_change":"2024-09-21T10:02:46.665711+0000","last_active":"2024-09-21T10:03:53.835262+0000","last_peered":"2024-09-21T10:03:53.835262+0000","last_clean":"2024-09-21T10:03:53.835262+0000","last_became_active":"2024-09-21T10:02:46.665165+0000","last_became_peered":"2024-09-21T10:02:46.665165+0000","last_unstale":"2024-09-21T10:03:53.835262+0000","last_undegraded":"2024-09-21T10:03:53.835262+0000","last_fullsized":"2024-09-21T10:03:53.835262+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:08.886554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839031+0000","last_change":"2024-09-21T10:03:53.838941+0000","last_active":"2024-09-21T10:03:53.839031+0000","last_peered":"2024-09-21T10:03:53.839031+0000","last_clean":"2024-09-21T10:03:53.839031+0000","last_became_active":"2024-09-21T10:03:53.838646+0000","last_became_peered":"2024-09-21T10:03:53.838646+0000","last_unstale":"2024-09-21T10:03:53.839031+0000","last_undegraded":"2024-09-21T10:03:53.839031+0000","last_fullsized":"2024-09-21T10:03:53.839031+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:25.167057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830074+0000","last_change":"2024-09-21T10:02:46.661363+0000","last_active":"2024-09-21T10:03:53.830074+0000","last_peered":"2024-09-21T10:03:53.830074+0000","last_clean":"2024-09-21T10:03:53.830074+0000","last_became_active":"2024-09-21T10:02:46.661005+0000","last_became_peered":"2024-09-21T10:02:46.661005+0000","last_unstale":"2024-09-21T10:03:53.830074+0000","last_undegraded":"2024-09-21T10:03:53.830074+0000","last_fullsized":"2024-09-21T10:03:53.830074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:42.908471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":349,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.838167+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:03:53.838167+0000","last_peered":"2024-09-21T10:03:53.838167+0000","last_clean":"2024-09-21T10:03:53.838167+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:03:53.838167+0000","last_undegraded":"2024-09-21T10:03:53.838167+0000","last_fullsized":"2024-09-21T10:03:53.838167+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.829108+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:03:52.829108+0000","last_peered":"2024-09-21T10:03:52.829108+0000","last_clean":"2024-09-21T10:03:52.829108+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:03:52.829108+0000","last_undegraded":"2024-09-21T10:03:52.829108+0000","last_fullsized":"2024-09-21T10:03:52.829108+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832815+0000","last_change":"2024-09-21T10:02:46.667831+0000","last_active":"2024-09-21T10:03:53.832815+0000","last_peered":"2024-09-21T10:03:53.832815+0000","last_clean":"2024-09-21T10:03:53.832815+0000","last_became_active":"2024-09-21T10:02:46.667507+0000","last_became_peered":"2024-09-21T10:02:46.667507+0000","last_unstale":"2024-09-21T10:03:53.832815+0000","last_undegraded":"2024-09-21T10:03:53.832815+0000","last_fullsized":"2024-09-21T10:03:53.832815+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:23.990802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832211+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:03:53.832211+0000","last_peered":"2024-09-21T10:03:53.832211+0000","last_clean":"2024-09-21T10:03:53.832211+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:03:53.832211+0000","last_undegraded":"2024-09-21T10:03:53.832211+0000","last_fullsized":"2024-09-21T10:03:53.832211+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836450+0000","last_change":"2024-09-21T10:03:19.767147+0000","last_active":"2024-09-21T10:03:52.836450+0000","last_peered":"2024-09-21T10:03:52.836450+0000","last_clean":"2024-09-21T10:03:52.836450+0000","last_became_active":"2024-09-21T10:03:19.766822+0000","last_became_peered":"2024-09-21T10:03:19.766822+0000","last_unstale":"2024-09-21T10:03:52.836450+0000","last_undegraded":"2024-09-21T10:03:52.836450+0000","last_fullsized":"2024-09-21T10:03:52.836450+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:38.875912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.841826+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:03:53.841826+0000","last_peered":"2024-09-21T10:03:53.841826+0000","last_clean":"2024-09-21T10:03:53.841826+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:03:53.841826+0000","last_undegraded":"2024-09-21T10:03:53.841826+0000","last_fullsized":"2024-09-21T10:03:53.841826+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836449+0000","last_change":"2024-09-21T10:02:46.659383+0000","last_active":"2024-09-21T10:03:52.836449+0000","last_peered":"2024-09-21T10:03:52.836449+0000","last_clean":"2024-09-21T10:03:52.836449+0000","last_became_active":"2024-09-21T10:02:46.659011+0000","last_became_peered":"2024-09-21T10:02:46.659011+0000","last_unstale":"2024-09-21T10:03:52.836449+0000","last_undegraded":"2024-09-21T10:03:52.836449+0000","last_fullsized":"2024-09-21T10:03:52.836449+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:04.650699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845276+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:03:53.845276+0000","last_peered":"2024-09-21T10:03:53.845276+0000","last_clean":"2024-09-21T10:03:53.845276+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:03:53.845276+0000","last_undegraded":"2024-09-21T10:03:53.845276+0000","last_fullsized":"2024-09-21T10:03:53.845276+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.844990+0000","last_change":"2024-09-21T10:03:53.844673+0000","last_active":"2024-09-21T10:03:53.844990+0000","last_peered":"2024-09-21T10:03:53.844990+0000","last_clean":"2024-09-21T10:03:53.844990+0000","last_became_active":"2024-09-21T10:03:53.844313+0000","last_became_peered":"2024-09-21T10:03:53.844313+0000","last_unstale":"2024-09-21T10:03:53.844990+0000","last_undegraded":"2024-09-21T10:03:53.844990+0000","last_fullsized":"2024-09-21T10:03:53.844990+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:42.529301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.841503+0000","last_change":"2024-09-21T10:03:53.841417+0000","last_active":"2024-09-21T10:03:53.841503+0000","last_peered":"2024-09-21T10:03:53.841503+0000","last_clean":"2024-09-21T10:03:53.841503+0000","last_became_active":"2024-09-21T10:03:53.840993+0000","last_became_peered":"2024-09-21T10:03:53.840993+0000","last_unstale":"2024-09-21T10:03:53.841503+0000","last_undegraded":"2024-09-21T10:03:53.841503+0000","last_fullsized":"2024-09-21T10:03:53.841503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:05.666490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":81,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.834009+0000","last_change":"2024-09-21T10:03:19.765682+0000","last_active":"2024-09-21T10:03:52.834009+0000","last_peered":"2024-09-21T10:03:52.834009+0000","last_clean":"2024-09-21T10:03:52.834009+0000","last_became_active":"2024-09-21T10:03:19.765344+0000","last_became_peered":"2024-09-21T10:03:19.765344+0000","last_unstale":"2024-09-21T10:03:52.834009+0000","last_undegraded":"2024-09-21T10:03:52.834009+0000","last_fullsized":"2024-09-21T10:03:52.834009+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:43.787280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839452+0000","last_change":"2024-09-21T10:03:53.839372+0000","last_active":"2024-09-21T10:03:53.839452+0000","last_peered":"2024-09-21T10:03:53.839452+0000","last_clean":"2024-09-21T10:03:53.839452+0000","last_became_active":"2024-09-21T10:03:53.839053+0000","last_became_peered":"2024-09-21T10:03:53.839053+0000","last_unstale":"2024-09-21T10:03:53.839452+0000","last_undegraded":"2024-09-21T10:03:53.839452+0000","last_fullsized":"2024-09-21T10:03:53.839452+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:29.647982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845364+0000","last_change":"2024-09-21T10:03:53.844946+0000","last_active":"2024-09-21T10:03:53.845364+0000","last_peered":"2024-09-21T10:03:53.845364+0000","last_clean":"2024-09-21T10:03:53.845364+0000","last_became_active":"2024-09-21T10:03:53.844613+0000","last_became_peered":"2024-09-21T10:03:53.844613+0000","last_unstale":"2024-09-21T10:03:53.845364+0000","last_undegraded":"2024-09-21T10:03:53.845364+0000","last_fullsized":"2024-09-21T10:03:53.845364+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:41.055573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.847034+0000","last_change":"2024-09-21T10:03:53.846948+0000","last_active":"2024-09-21T10:03:53.847034+0000","last_peered":"2024-09-21T10:03:53.847034+0000","last_clean":"2024-09-21T10:03:53.847034+0000","last_became_active":"2024-09-21T10:03:53.846614+0000","last_became_peered":"2024-09-21T10:03:53.846614+0000","last_unstale":"2024-09-21T10:03:53.847034+0000","last_undegraded":"2024-09-21T10:03:53.847034+0000","last_fullsized":"2024-09-21T10:03:53.847034+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:53.968372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845285+0000","last_change":"2024-09-21T10:03:53.845201+0000","last_active":"2024-09-21T10:03:53.845285+0000","last_peered":"2024-09-21T10:03:53.845285+0000","last_clean":"2024-09-21T10:03:53.845285+0000","last_became_active":"2024-09-21T10:03:53.843822+0000","last_became_peered":"2024-09-21T10:03:53.843822+0000","last_unstale":"2024-09-21T10:03:53.845285+0000","last_undegraded":"2024-09-21T10:03:53.845285+0000","last_fullsized":"2024-09-21T10:03:53.845285+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:28.895373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.828363+0000","last_change":"2024-09-21T10:03:19.768025+0000","last_active":"2024-09-21T10:03:52.828363+0000","last_peered":"2024-09-21T10:03:52.828363+0000","last_clean":"2024-09-21T10:03:52.828363+0000","last_became_active":"2024-09-21T10:03:19.767632+0000","last_became_peered":"2024-09-21T10:03:19.767632+0000","last_unstale":"2024-09-21T10:03:52.828363+0000","last_undegraded":"2024-09-21T10:03:52.828363+0000","last_fullsized":"2024-09-21T10:03:52.828363+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:40.023759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840358+0000","last_change":"2024-09-21T10:03:53.840263+0000","last_active":"2024-09-21T10:03:53.840358+0000","last_peered":"2024-09-21T10:03:53.840358+0000","last_clean":"2024-09-21T10:03:53.840358+0000","last_became_active":"2024-09-21T10:03:53.839894+0000","last_became_peered":"2024-09-21T10:03:53.839894+0000","last_unstale":"2024-09-21T10:03:53.840358+0000","last_undegraded":"2024-09-21T10:03:53.840358+0000","last_fullsized":"2024-09-21T10:03:53.840358+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:52.489580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830275+0000","last_change":"2024-09-21T10:02:12.846054+0000","last_active":"2024-09-21T10:03:53.830275+0000","last_peered":"2024-09-21T10:03:53.830275+0000","last_clean":"2024-09-21T10:03:53.830275+0000","last_became_active":"2024-09-21T10:02:12.843696+0000","last_became_peered":"2024-09-21T10:02:12.843696+0000","last_unstale":"2024-09-21T10:03:53.830275+0000","last_undegraded":"2024-09-21T10:03:53.830275+0000","last_fullsized":"2024-09-21T10:03:53.830275+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:32.280697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840992+0000","last_change":"2024-09-21T10:03:53.840992+0000","last_active":"2024-09-21T10:03:53.840992+0000","last_peered":"2024-09-21T10:03:53.840992+0000","last_clean":"2024-09-21T10:03:53.840992+0000","last_became_active":"2024-09-21T10:03:53.840551+0000","last_became_peered":"2024-09-21T10:03:53.840551+0000","last_unstale":"2024-09-21T10:03:53.840992+0000","last_undegraded":"2024-09-21T10:03:53.840992+0000","last_fullsized":"2024-09-21T10:03:53.840992+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:04.442126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836717+0000","last_change":"2024-09-21T10:02:46.659513+0000","last_active":"2024-09-21T10:03:52.836717+0000","last_peered":"2024-09-21T10:03:52.836717+0000","last_clean":"2024-09-21T10:03:52.836717+0000","last_became_active":"2024-09-21T10:02:46.659130+0000","last_became_peered":"2024-09-21T10:02:46.659130+0000","last_unstale":"2024-09-21T10:03:52.836717+0000","last_undegraded":"2024-09-21T10:03:52.836717+0000","last_fullsized":"2024-09-21T10:03:52.836717+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:24.792175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835006+0000","last_change":"2024-09-21T10:03:53.835006+0000","last_active":"2024-09-21T10:03:53.835006+0000","last_peered":"2024-09-21T10:03:53.835006+0000","last_clean":"2024-09-21T10:03:53.835006+0000","last_became_active":"2024-09-21T10:03:53.834608+0000","last_became_peered":"2024-09-21T10:03:53.834608+0000","last_unstale":"2024-09-21T10:03:53.835006+0000","last_undegraded":"2024-09-21T10:03:53.835006+0000","last_fullsized":"2024-09-21T10:03:53.835006+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:56.108606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":59,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.837866+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:03:52.837866+0000","last_peered":"2024-09-21T10:03:52.837866+0000","last_clean":"2024-09-21T10:03:52.837866+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:03:52.837866+0000","last_undegraded":"2024-09-21T10:03:52.837866+0000","last_fullsized":"2024-09-21T10:03:52.837866+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.844128+0000","last_change":"2024-09-21T10:03:53.844043+0000","last_active":"2024-09-21T10:03:53.844128+0000","last_peered":"2024-09-21T10:03:53.844128+0000","last_clean":"2024-09-21T10:03:53.844128+0000","last_became_active":"2024-09-21T10:03:53.842891+0000","last_became_peered":"2024-09-21T10:03:53.842891+0000","last_unstale":"2024-09-21T10:03:53.844128+0000","last_undegraded":"2024-09-21T10:03:53.844128+0000","last_fullsized":"2024-09-21T10:03:53.844128+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:39.891977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842964+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:03:53.842964+0000","last_peered":"2024-09-21T10:03:53.842964+0000","last_clean":"2024-09-21T10:03:53.842964+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:03:53.842964+0000","last_undegraded":"2024-09-21T10:03:53.842964+0000","last_fullsized":"2024-09-21T10:03:53.842964+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.828174+0000","last_change":"2024-09-21T10:02:46.659818+0000","last_active":"2024-09-21T10:03:52.828174+0000","last_peered":"2024-09-21T10:03:52.828174+0000","last_clean":"2024-09-21T10:03:52.828174+0000","last_became_active":"2024-09-21T10:02:46.659457+0000","last_became_peered":"2024-09-21T10:02:46.659457+0000","last_unstale":"2024-09-21T10:03:52.828174+0000","last_undegraded":"2024-09-21T10:03:52.828174+0000","last_fullsized":"2024-09-21T10:03:52.828174+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:21.962330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830391+0000","last_change":"2024-09-21T10:03:19.769549+0000","last_active":"2024-09-21T10:03:53.830391+0000","last_peered":"2024-09-21T10:03:53.830391+0000","last_clean":"2024-09-21T10:03:53.830391+0000","last_became_active":"2024-09-21T10:03:19.768854+0000","last_became_peered":"2024-09-21T10:03:19.768854+0000","last_unstale":"2024-09-21T10:03:53.830391+0000","last_undegraded":"2024-09-21T10:03:53.830391+0000","last_fullsized":"2024-09-21T10:03:53.830391+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:29.415039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234181,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37008,"kb_used_data":5884,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708144,"statfs":{"total":95995035648,"available":95957139456,"internally_reserved":0,"allocated":6025216,"data_stored":5417204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495819,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32908,"kb_used_data":1784,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712244,"statfs":{"total":95995035648,"available":95961337856,"internally_reserved":0,"allocated":1826816,"data_stored":1223104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757456,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28216,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716936,"statfs":{"total":95995035648,"available":95966142464,"internally_reserved":0,"allocated":1224704,"data_stored":628562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019092,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32332,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712820,"statfs":{"total":95995035648,"available":95961927680,"internally_reserved":0,"allocated":1236992,"data_stored":632793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435277,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32888,"kb_used_data":1188,"kb_used_omap":5,"kb_used_meta":31674,"kb_avail":93712264,"statfs":{"total":95995035648,"available":95961358336,"internally_reserved":0,"allocated":1216512,"data_stored":627993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":32434216},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598804,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1188,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1216512,"data_stored":628211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":28172749},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762330,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36976,"kb_used_data":5280,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708176,"statfs":{"total":95995035648,"available":95957172224,"internally_reserved":0,"allocated":5406720,"data_stored":4821953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":32432381},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925857,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33404,"kb_used_data":5868,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711748,"statfs":{"total":95995035648,"available":95960829952,"internally_reserved":0,"allocated":6008832,"data_stored":5413179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":28172478},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:04:50.382 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T10:04:50.382 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T10:04:50.382 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T10:04:50.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T10:04:50.591 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:50.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:50 smithi032 ceph-mon[101299]: from='client.44468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:50.934 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:50 smithi032 ceph-mon[105748]: from='client.44468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:50 smithi045 ceph-mon[95976]: from='client.44468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:51.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:51 smithi032 ceph-mon[105748]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:51.688 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:51 smithi032 ceph-mon[101299]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:51 smithi045 ceph-mon[95976]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:52.465 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:04:52.465 INFO:teuthology.orchestra.run.smithi032.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T10:04:52.837 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:52 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/62865415' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-21T10:04:52.837 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:52 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/62865415' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-21T10:04:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:52 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/62865415' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-21T10:04:53.135 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T10:04:53.136 INFO:teuthology.run_tasks:Running task print... 2024-09-21T10:04:53.145 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-09-21T10:04:53.145 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T10:04:53.178 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T10:04:53.179 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T10:04:53.179 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-09-21T10:04:53.180 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-21T10:04:53.181 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T10:04:53.181 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-21T10:04:53.181 INFO:tasks.thrashosds:Beginning thrashosds... 2024-09-21T10:04:53.181 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T10:04:53.181 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:04:53.389 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:53.824 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:53 smithi032 ceph-mon[105748]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:53.824 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:53 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:53.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:53 smithi032 ceph-mon[101299]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:53.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:53 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:54.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:53 smithi045 ceph-mon[95976]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:53 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:55.190 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:04:55.191 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:04:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:55 smithi045 ceph-mon[95976]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:55.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:55.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:55.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:55 smithi032 ceph-mon[101299]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:55.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:55.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:55.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:55 smithi032 ceph-mon[105748]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:55.927 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":281,"stamp":"2024-09-21T10:04:54.124228+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":262452,"kb_used_data":23596,"kb_used_omap":35,"kb_used_meta":238684,"kb_avail":749698764,"statfs":{"total":767960285184,"available":767691534336,"internally_reserved":0,"allocated":24162304,"data_stored":19392999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":244413134},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002926"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843669+0000","last_change":"2024-09-21T10:03:53.843548+0000","last_active":"2024-09-21T10:03:53.843669+0000","last_peered":"2024-09-21T10:03:53.843669+0000","last_clean":"2024-09-21T10:03:53.843669+0000","last_became_active":"2024-09-21T10:03:53.843182+0000","last_became_peered":"2024-09-21T10:03:53.843182+0000","last_unstale":"2024-09-21T10:03:53.843669+0000","last_undegraded":"2024-09-21T10:03:53.843669+0000","last_fullsized":"2024-09-21T10:03:53.843669+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:09.464209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834897+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:03:53.834897+0000","last_peered":"2024-09-21T10:03:53.834897+0000","last_clean":"2024-09-21T10:03:53.834897+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:03:53.834897+0000","last_undegraded":"2024-09-21T10:03:53.834897+0000","last_fullsized":"2024-09-21T10:03:53.834897+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840530+0000","last_change":"2024-09-21T10:03:53.839987+0000","last_active":"2024-09-21T10:03:53.840530+0000","last_peered":"2024-09-21T10:03:53.840530+0000","last_clean":"2024-09-21T10:03:53.840530+0000","last_became_active":"2024-09-21T10:03:53.839664+0000","last_became_peered":"2024-09-21T10:03:53.839664+0000","last_unstale":"2024-09-21T10:03:53.840530+0000","last_undegraded":"2024-09-21T10:03:53.840530+0000","last_fullsized":"2024-09-21T10:03:53.840530+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:51.726392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837862+0000","last_change":"2024-09-21T10:03:53.837496+0000","last_active":"2024-09-21T10:03:53.837862+0000","last_peered":"2024-09-21T10:03:53.837862+0000","last_clean":"2024-09-21T10:03:53.837862+0000","last_became_active":"2024-09-21T10:03:53.837166+0000","last_became_peered":"2024-09-21T10:03:53.837166+0000","last_unstale":"2024-09-21T10:03:53.837862+0000","last_undegraded":"2024-09-21T10:03:53.837862+0000","last_fullsized":"2024-09-21T10:03:53.837862+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:45.795571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845452+0000","last_change":"2024-09-21T10:03:53.844990+0000","last_active":"2024-09-21T10:03:53.845452+0000","last_peered":"2024-09-21T10:03:53.845452+0000","last_clean":"2024-09-21T10:03:53.845452+0000","last_became_active":"2024-09-21T10:03:53.844664+0000","last_became_peered":"2024-09-21T10:03:53.844664+0000","last_unstale":"2024-09-21T10:03:53.845452+0000","last_undegraded":"2024-09-21T10:03:53.845452+0000","last_fullsized":"2024-09-21T10:03:53.845452+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:01.031522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835247+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:03:53.835247+0000","last_peered":"2024-09-21T10:03:53.835247+0000","last_clean":"2024-09-21T10:03:53.835247+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:03:53.835247+0000","last_undegraded":"2024-09-21T10:03:53.835247+0000","last_fullsized":"2024-09-21T10:03:53.835247+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845872+0000","last_change":"2024-09-21T10:03:53.845785+0000","last_active":"2024-09-21T10:03:53.845872+0000","last_peered":"2024-09-21T10:03:53.845872+0000","last_clean":"2024-09-21T10:03:53.845872+0000","last_became_active":"2024-09-21T10:03:53.845303+0000","last_became_peered":"2024-09-21T10:03:53.845303+0000","last_unstale":"2024-09-21T10:03:53.845872+0000","last_undegraded":"2024-09-21T10:03:53.845872+0000","last_fullsized":"2024-09-21T10:03:53.845872+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:04.976007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837943+0000","last_change":"2024-09-21T10:02:12.838786+0000","last_active":"2024-09-21T10:03:53.837943+0000","last_peered":"2024-09-21T10:03:53.837943+0000","last_clean":"2024-09-21T10:03:53.837943+0000","last_became_active":"2024-09-21T10:02:12.838242+0000","last_became_peered":"2024-09-21T10:02:12.838242+0000","last_unstale":"2024-09-21T10:03:53.837943+0000","last_undegraded":"2024-09-21T10:03:53.837943+0000","last_fullsized":"2024-09-21T10:03:53.837943+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:31.071136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835170+0000","last_change":"2024-09-21T10:02:12.840334+0000","last_active":"2024-09-21T10:03:53.835170+0000","last_peered":"2024-09-21T10:03:53.835170+0000","last_clean":"2024-09-21T10:03:53.835170+0000","last_became_active":"2024-09-21T10:02:12.839928+0000","last_became_peered":"2024-09-21T10:02:12.839928+0000","last_unstale":"2024-09-21T10:03:53.835170+0000","last_undegraded":"2024-09-21T10:03:53.835170+0000","last_fullsized":"2024-09-21T10:03:53.835170+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:38.593644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836136+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T10:03:52.836136+0000","last_peered":"2024-09-21T10:03:52.836136+0000","last_clean":"2024-09-21T10:03:52.836136+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T10:03:52.836136+0000","last_undegraded":"2024-09-21T10:03:52.836136+0000","last_fullsized":"2024-09-21T10:03:52.836136+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.836116+0000","last_change":"2024-09-21T10:02:46.662169+0000","last_active":"2024-09-21T10:03:53.836116+0000","last_peered":"2024-09-21T10:03:53.836116+0000","last_clean":"2024-09-21T10:03:53.836116+0000","last_became_active":"2024-09-21T10:02:46.661721+0000","last_became_peered":"2024-09-21T10:02:46.661721+0000","last_unstale":"2024-09-21T10:03:53.836116+0000","last_undegraded":"2024-09-21T10:03:53.836116+0000","last_fullsized":"2024-09-21T10:03:53.836116+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:10.414152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840738+0000","last_change":"2024-09-21T10:03:53.840738+0000","last_active":"2024-09-21T10:03:53.840738+0000","last_peered":"2024-09-21T10:03:53.840738+0000","last_clean":"2024-09-21T10:03:53.840738+0000","last_became_active":"2024-09-21T10:03:53.840363+0000","last_became_peered":"2024-09-21T10:03:53.840363+0000","last_unstale":"2024-09-21T10:03:53.840738+0000","last_undegraded":"2024-09-21T10:03:53.840738+0000","last_fullsized":"2024-09-21T10:03:53.840738+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:26.371181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835739+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T10:03:53.835739+0000","last_peered":"2024-09-21T10:03:53.835739+0000","last_clean":"2024-09-21T10:03:53.835739+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T10:03:53.835739+0000","last_undegraded":"2024-09-21T10:03:53.835739+0000","last_fullsized":"2024-09-21T10:03:53.835739+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833635+0000","last_change":"2024-09-21T10:02:46.663127+0000","last_active":"2024-09-21T10:03:52.833635+0000","last_peered":"2024-09-21T10:03:52.833635+0000","last_clean":"2024-09-21T10:03:52.833635+0000","last_became_active":"2024-09-21T10:02:46.662791+0000","last_became_peered":"2024-09-21T10:02:46.662791+0000","last_unstale":"2024-09-21T10:03:52.833635+0000","last_undegraded":"2024-09-21T10:03:52.833635+0000","last_fullsized":"2024-09-21T10:03:52.833635+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:59.990122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843912+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:03:53.843912+0000","last_peered":"2024-09-21T10:03:53.843912+0000","last_clean":"2024-09-21T10:03:53.843912+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:03:53.843912+0000","last_undegraded":"2024-09-21T10:03:53.843912+0000","last_fullsized":"2024-09-21T10:03:53.843912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830367+0000","last_change":"2024-09-21T10:02:46.662651+0000","last_active":"2024-09-21T10:03:53.830367+0000","last_peered":"2024-09-21T10:03:53.830367+0000","last_clean":"2024-09-21T10:03:53.830367+0000","last_became_active":"2024-09-21T10:02:46.662304+0000","last_became_peered":"2024-09-21T10:02:46.662304+0000","last_unstale":"2024-09-21T10:03:53.830367+0000","last_undegraded":"2024-09-21T10:03:53.830367+0000","last_fullsized":"2024-09-21T10:03:53.830367+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:06.746929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840595+0000","last_change":"2024-09-21T10:03:53.840511+0000","last_active":"2024-09-21T10:03:53.840595+0000","last_peered":"2024-09-21T10:03:53.840595+0000","last_clean":"2024-09-21T10:03:53.840595+0000","last_became_active":"2024-09-21T10:03:53.840136+0000","last_became_peered":"2024-09-21T10:03:53.840136+0000","last_unstale":"2024-09-21T10:03:53.840595+0000","last_undegraded":"2024-09-21T10:03:53.840595+0000","last_fullsized":"2024-09-21T10:03:53.840595+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:24.765990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840155+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:03:53.840155+0000","last_peered":"2024-09-21T10:03:53.840155+0000","last_clean":"2024-09-21T10:03:53.840155+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:03:53.840155+0000","last_undegraded":"2024-09-21T10:03:53.840155+0000","last_fullsized":"2024-09-21T10:03:53.840155+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833841+0000","last_change":"2024-09-21T10:02:46.663900+0000","last_active":"2024-09-21T10:03:52.833841+0000","last_peered":"2024-09-21T10:03:52.833841+0000","last_clean":"2024-09-21T10:03:52.833841+0000","last_became_active":"2024-09-21T10:02:46.663594+0000","last_became_peered":"2024-09-21T10:02:46.663594+0000","last_unstale":"2024-09-21T10:03:52.833841+0000","last_undegraded":"2024-09-21T10:03:52.833841+0000","last_fullsized":"2024-09-21T10:03:52.833841+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:24.140061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832354+0000","last_change":"2024-09-21T10:03:19.765623+0000","last_active":"2024-09-21T10:03:53.832354+0000","last_peered":"2024-09-21T10:03:53.832354+0000","last_clean":"2024-09-21T10:03:53.832354+0000","last_became_active":"2024-09-21T10:03:19.765227+0000","last_became_peered":"2024-09-21T10:03:19.765227+0000","last_unstale":"2024-09-21T10:03:53.832354+0000","last_undegraded":"2024-09-21T10:03:53.832354+0000","last_fullsized":"2024-09-21T10:03:53.832354+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:52.227826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.829335+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:03:52.829335+0000","last_peered":"2024-09-21T10:03:52.829335+0000","last_clean":"2024-09-21T10:03:52.829335+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:03:52.829335+0000","last_undegraded":"2024-09-21T10:03:52.829335+0000","last_fullsized":"2024-09-21T10:03:52.829335+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832211+0000","last_change":"2024-09-21T10:02:46.664014+0000","last_active":"2024-09-21T10:03:53.832211+0000","last_peered":"2024-09-21T10:03:53.832211+0000","last_clean":"2024-09-21T10:03:53.832211+0000","last_became_active":"2024-09-21T10:02:46.663640+0000","last_became_peered":"2024-09-21T10:02:46.663640+0000","last_unstale":"2024-09-21T10:03:53.832211+0000","last_undegraded":"2024-09-21T10:03:53.832211+0000","last_fullsized":"2024-09-21T10:03:53.832211+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:03.733462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845713+0000","last_change":"2024-09-21T10:03:53.845620+0000","last_active":"2024-09-21T10:03:53.845713+0000","last_peered":"2024-09-21T10:03:53.845713+0000","last_clean":"2024-09-21T10:03:53.845713+0000","last_became_active":"2024-09-21T10:03:53.845290+0000","last_became_peered":"2024-09-21T10:03:53.845290+0000","last_unstale":"2024-09-21T10:03:53.845713+0000","last_undegraded":"2024-09-21T10:03:53.845713+0000","last_fullsized":"2024-09-21T10:03:53.845713+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:20.076623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.846700+0000","last_change":"2024-09-21T10:03:53.846606+0000","last_active":"2024-09-21T10:03:53.846700+0000","last_peered":"2024-09-21T10:03:53.846700+0000","last_clean":"2024-09-21T10:03:53.846700+0000","last_became_active":"2024-09-21T10:03:53.846245+0000","last_became_peered":"2024-09-21T10:03:53.846245+0000","last_unstale":"2024-09-21T10:03:53.846700+0000","last_undegraded":"2024-09-21T10:03:53.846700+0000","last_fullsized":"2024-09-21T10:03:53.846700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:20.745470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":405,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835020+0000","last_change":"2024-09-21T10:02:12.840225+0000","last_active":"2024-09-21T10:03:53.835020+0000","last_peered":"2024-09-21T10:03:53.835020+0000","last_clean":"2024-09-21T10:03:53.835020+0000","last_became_active":"2024-09-21T10:02:12.839813+0000","last_became_peered":"2024-09-21T10:02:12.839813+0000","last_unstale":"2024-09-21T10:03:53.835020+0000","last_undegraded":"2024-09-21T10:03:53.835020+0000","last_fullsized":"2024-09-21T10:03:53.835020+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:36.630881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":423,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836626+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:03:52.836626+0000","last_peered":"2024-09-21T10:03:52.836626+0000","last_clean":"2024-09-21T10:03:52.836626+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:03:52.836626+0000","last_undegraded":"2024-09-21T10:03:52.836626+0000","last_fullsized":"2024-09-21T10:03:52.836626+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843425+0000","last_change":"2024-09-21T10:03:53.843344+0000","last_active":"2024-09-21T10:03:53.843425+0000","last_peered":"2024-09-21T10:03:53.843425+0000","last_clean":"2024-09-21T10:03:53.843425+0000","last_became_active":"2024-09-21T10:03:53.842304+0000","last_became_peered":"2024-09-21T10:03:53.842304+0000","last_unstale":"2024-09-21T10:03:53.843425+0000","last_undegraded":"2024-09-21T10:03:53.843425+0000","last_fullsized":"2024-09-21T10:03:53.843425+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:31.221735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833226+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:03:52.833226+0000","last_peered":"2024-09-21T10:03:52.833226+0000","last_clean":"2024-09-21T10:03:52.833226+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:03:52.833226+0000","last_undegraded":"2024-09-21T10:03:52.833226+0000","last_fullsized":"2024-09-21T10:03:52.833226+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":568,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.832252+0000","last_change":"2024-09-21T10:02:46.663394+0000","last_active":"2024-09-21T10:03:52.832252+0000","last_peered":"2024-09-21T10:03:52.832252+0000","last_clean":"2024-09-21T10:03:52.832252+0000","last_became_active":"2024-09-21T10:02:46.662879+0000","last_became_peered":"2024-09-21T10:02:46.662879+0000","last_unstale":"2024-09-21T10:03:52.832252+0000","last_undegraded":"2024-09-21T10:03:52.832252+0000","last_fullsized":"2024-09-21T10:03:52.832252+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.125795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835797+0000","last_change":"2024-09-21T10:03:53.835329+0000","last_active":"2024-09-21T10:03:53.835797+0000","last_peered":"2024-09-21T10:03:53.835797+0000","last_clean":"2024-09-21T10:03:53.835797+0000","last_became_active":"2024-09-21T10:03:53.834934+0000","last_became_peered":"2024-09-21T10:03:53.834934+0000","last_unstale":"2024-09-21T10:03:53.835797+0000","last_undegraded":"2024-09-21T10:03:53.835797+0000","last_fullsized":"2024-09-21T10:03:53.835797+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:38.355817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830913+0000","last_change":"2024-09-21T10:03:19.770016+0000","last_active":"2024-09-21T10:03:53.830913+0000","last_peered":"2024-09-21T10:03:53.830913+0000","last_clean":"2024-09-21T10:03:53.830913+0000","last_became_active":"2024-09-21T10:03:19.768866+0000","last_became_peered":"2024-09-21T10:03:19.768866+0000","last_unstale":"2024-09-21T10:03:53.830913+0000","last_undegraded":"2024-09-21T10:03:53.830913+0000","last_fullsized":"2024-09-21T10:03:53.830913+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:25.068940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":573,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840408+0000","last_change":"2024-09-21T10:03:53.840322+0000","last_active":"2024-09-21T10:03:53.840408+0000","last_peered":"2024-09-21T10:03:53.840408+0000","last_clean":"2024-09-21T10:03:53.840408+0000","last_became_active":"2024-09-21T10:03:53.839753+0000","last_became_peered":"2024-09-21T10:03:53.839753+0000","last_unstale":"2024-09-21T10:03:53.840408+0000","last_undegraded":"2024-09-21T10:03:53.840408+0000","last_fullsized":"2024-09-21T10:03:53.840408+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:26.364888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835436+0000","last_change":"2024-09-21T10:03:19.765402+0000","last_active":"2024-09-21T10:03:53.835436+0000","last_peered":"2024-09-21T10:03:53.835436+0000","last_clean":"2024-09-21T10:03:53.835436+0000","last_became_active":"2024-09-21T10:03:19.765036+0000","last_became_peered":"2024-09-21T10:03:19.765036+0000","last_unstale":"2024-09-21T10:03:53.835436+0000","last_undegraded":"2024-09-21T10:03:53.835436+0000","last_fullsized":"2024-09-21T10:03:53.835436+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:32.230377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837881+0000","last_change":"2024-09-21T10:02:46.660814+0000","last_active":"2024-09-21T10:03:53.837881+0000","last_peered":"2024-09-21T10:03:53.837881+0000","last_clean":"2024-09-21T10:03:53.837881+0000","last_became_active":"2024-09-21T10:02:46.660325+0000","last_became_peered":"2024-09-21T10:02:46.660325+0000","last_unstale":"2024-09-21T10:03:53.837881+0000","last_undegraded":"2024-09-21T10:03:53.837881+0000","last_fullsized":"2024-09-21T10:03:53.837881+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:44.427243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":811,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836665+0000","last_change":"2024-09-21T10:02:46.657107+0000","last_active":"2024-09-21T10:03:52.836665+0000","last_peered":"2024-09-21T10:03:52.836665+0000","last_clean":"2024-09-21T10:03:52.836665+0000","last_became_active":"2024-09-21T10:02:46.656633+0000","last_became_peered":"2024-09-21T10:02:46.656633+0000","last_unstale":"2024-09-21T10:03:52.836665+0000","last_undegraded":"2024-09-21T10:03:52.836665+0000","last_fullsized":"2024-09-21T10:03:52.836665+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:42:18.534832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.830018+0000","last_change":"2024-09-21T10:02:12.840581+0000","last_active":"2024-09-21T10:03:52.830018+0000","last_peered":"2024-09-21T10:03:52.830018+0000","last_clean":"2024-09-21T10:03:52.830018+0000","last_became_active":"2024-09-21T10:02:12.840033+0000","last_became_peered":"2024-09-21T10:02:12.840033+0000","last_unstale":"2024-09-21T10:03:52.830018+0000","last_undegraded":"2024-09-21T10:03:52.830018+0000","last_fullsized":"2024-09-21T10:03:52.830018+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:22.449586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843937+0000","last_change":"2024-09-21T10:03:53.843551+0000","last_active":"2024-09-21T10:03:53.843937+0000","last_peered":"2024-09-21T10:03:53.843937+0000","last_clean":"2024-09-21T10:03:53.843937+0000","last_became_active":"2024-09-21T10:03:53.843007+0000","last_became_peered":"2024-09-21T10:03:53.843007+0000","last_unstale":"2024-09-21T10:03:53.843937+0000","last_undegraded":"2024-09-21T10:03:53.843937+0000","last_fullsized":"2024-09-21T10:03:53.843937+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:11.757447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843567+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:03:53.843567+0000","last_peered":"2024-09-21T10:03:53.843567+0000","last_clean":"2024-09-21T10:03:53.843567+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:03:53.843567+0000","last_undegraded":"2024-09-21T10:03:53.843567+0000","last_fullsized":"2024-09-21T10:03:53.843567+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":530,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.836990+0000","last_change":"2024-09-21T10:02:46.662410+0000","last_active":"2024-09-21T10:03:53.836990+0000","last_peered":"2024-09-21T10:03:53.836990+0000","last_clean":"2024-09-21T10:03:53.836990+0000","last_became_active":"2024-09-21T10:02:46.661506+0000","last_became_peered":"2024-09-21T10:02:46.661506+0000","last_unstale":"2024-09-21T10:03:53.836990+0000","last_undegraded":"2024-09-21T10:03:53.836990+0000","last_fullsized":"2024-09-21T10:03:53.836990+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:40:33.566844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834945+0000","last_change":"2024-09-21T10:02:46.665713+0000","last_active":"2024-09-21T10:03:53.834945+0000","last_peered":"2024-09-21T10:03:53.834945+0000","last_clean":"2024-09-21T10:03:53.834945+0000","last_became_active":"2024-09-21T10:02:46.665316+0000","last_became_peered":"2024-09-21T10:02:46.665316+0000","last_unstale":"2024-09-21T10:03:53.834945+0000","last_undegraded":"2024-09-21T10:03:53.834945+0000","last_fullsized":"2024-09-21T10:03:53.834945+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.951202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837129+0000","last_change":"2024-09-21T10:03:19.768187+0000","last_active":"2024-09-21T10:03:53.837129+0000","last_peered":"2024-09-21T10:03:53.837129+0000","last_clean":"2024-09-21T10:03:53.837129+0000","last_became_active":"2024-09-21T10:03:19.767801+0000","last_became_peered":"2024-09-21T10:03:19.767801+0000","last_unstale":"2024-09-21T10:03:53.837129+0000","last_undegraded":"2024-09-21T10:03:53.837129+0000","last_fullsized":"2024-09-21T10:03:53.837129+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:38.042481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":511,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835831+0000","last_change":"2024-09-21T10:02:12.840296+0000","last_active":"2024-09-21T10:03:53.835831+0000","last_peered":"2024-09-21T10:03:53.835831+0000","last_clean":"2024-09-21T10:03:53.835831+0000","last_became_active":"2024-09-21T10:02:12.839881+0000","last_became_peered":"2024-09-21T10:02:12.839881+0000","last_unstale":"2024-09-21T10:03:53.835831+0000","last_undegraded":"2024-09-21T10:03:53.835831+0000","last_fullsized":"2024-09-21T10:03:53.835831+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:49.166865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830265+0000","last_change":"2024-09-21T10:02:12.845158+0000","last_active":"2024-09-21T10:03:53.830265+0000","last_peered":"2024-09-21T10:03:53.830265+0000","last_clean":"2024-09-21T10:03:53.830265+0000","last_became_active":"2024-09-21T10:02:12.843399+0000","last_became_peered":"2024-09-21T10:02:12.843399+0000","last_unstale":"2024-09-21T10:03:53.830265+0000","last_undegraded":"2024-09-21T10:03:53.830265+0000","last_fullsized":"2024-09-21T10:03:53.830265+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:27.552491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842700+0000","last_change":"2024-09-21T10:03:53.842610+0000","last_active":"2024-09-21T10:03:53.842700+0000","last_peered":"2024-09-21T10:03:53.842700+0000","last_clean":"2024-09-21T10:03:53.842700+0000","last_became_active":"2024-09-21T10:03:53.841888+0000","last_became_peered":"2024-09-21T10:03:53.841888+0000","last_unstale":"2024-09-21T10:03:53.842700+0000","last_undegraded":"2024-09-21T10:03:53.842700+0000","last_fullsized":"2024-09-21T10:03:53.842700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:05.833771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":635,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835262+0000","last_change":"2024-09-21T10:02:46.665711+0000","last_active":"2024-09-21T10:03:53.835262+0000","last_peered":"2024-09-21T10:03:53.835262+0000","last_clean":"2024-09-21T10:03:53.835262+0000","last_became_active":"2024-09-21T10:02:46.665165+0000","last_became_peered":"2024-09-21T10:02:46.665165+0000","last_unstale":"2024-09-21T10:03:53.835262+0000","last_undegraded":"2024-09-21T10:03:53.835262+0000","last_fullsized":"2024-09-21T10:03:53.835262+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:08.886554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839031+0000","last_change":"2024-09-21T10:03:53.838941+0000","last_active":"2024-09-21T10:03:53.839031+0000","last_peered":"2024-09-21T10:03:53.839031+0000","last_clean":"2024-09-21T10:03:53.839031+0000","last_became_active":"2024-09-21T10:03:53.838646+0000","last_became_peered":"2024-09-21T10:03:53.838646+0000","last_unstale":"2024-09-21T10:03:53.839031+0000","last_undegraded":"2024-09-21T10:03:53.839031+0000","last_fullsized":"2024-09-21T10:03:53.839031+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:25.167057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830074+0000","last_change":"2024-09-21T10:02:46.661363+0000","last_active":"2024-09-21T10:03:53.830074+0000","last_peered":"2024-09-21T10:03:53.830074+0000","last_clean":"2024-09-21T10:03:53.830074+0000","last_became_active":"2024-09-21T10:02:46.661005+0000","last_became_peered":"2024-09-21T10:02:46.661005+0000","last_unstale":"2024-09-21T10:03:53.830074+0000","last_undegraded":"2024-09-21T10:03:53.830074+0000","last_fullsized":"2024-09-21T10:03:53.830074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:42.908471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":349,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.838167+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:03:53.838167+0000","last_peered":"2024-09-21T10:03:53.838167+0000","last_clean":"2024-09-21T10:03:53.838167+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:03:53.838167+0000","last_undegraded":"2024-09-21T10:03:53.838167+0000","last_fullsized":"2024-09-21T10:03:53.838167+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.833021+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:03:53.833021+0000","last_peered":"2024-09-21T10:03:53.833021+0000","last_clean":"2024-09-21T10:03:53.833021+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:03:53.833021+0000","last_undegraded":"2024-09-21T10:03:53.833021+0000","last_fullsized":"2024-09-21T10:03:53.833021+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832815+0000","last_change":"2024-09-21T10:02:46.667831+0000","last_active":"2024-09-21T10:03:53.832815+0000","last_peered":"2024-09-21T10:03:53.832815+0000","last_clean":"2024-09-21T10:03:53.832815+0000","last_became_active":"2024-09-21T10:02:46.667507+0000","last_became_peered":"2024-09-21T10:02:46.667507+0000","last_unstale":"2024-09-21T10:03:53.832815+0000","last_undegraded":"2024-09-21T10:03:53.832815+0000","last_fullsized":"2024-09-21T10:03:53.832815+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:23.990802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832211+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:03:53.832211+0000","last_peered":"2024-09-21T10:03:53.832211+0000","last_clean":"2024-09-21T10:03:53.832211+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:03:53.832211+0000","last_undegraded":"2024-09-21T10:03:53.832211+0000","last_fullsized":"2024-09-21T10:03:53.832211+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836450+0000","last_change":"2024-09-21T10:03:19.767147+0000","last_active":"2024-09-21T10:03:52.836450+0000","last_peered":"2024-09-21T10:03:52.836450+0000","last_clean":"2024-09-21T10:03:52.836450+0000","last_became_active":"2024-09-21T10:03:19.766822+0000","last_became_peered":"2024-09-21T10:03:19.766822+0000","last_unstale":"2024-09-21T10:03:52.836450+0000","last_undegraded":"2024-09-21T10:03:52.836450+0000","last_fullsized":"2024-09-21T10:03:52.836450+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:38.875912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842074+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:03:53.842074+0000","last_peered":"2024-09-21T10:03:53.842074+0000","last_clean":"2024-09-21T10:03:53.842074+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:03:53.842074+0000","last_undegraded":"2024-09-21T10:03:53.842074+0000","last_fullsized":"2024-09-21T10:03:53.842074+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834141+0000","last_change":"2024-09-21T10:02:46.659383+0000","last_active":"2024-09-21T10:03:53.834141+0000","last_peered":"2024-09-21T10:03:53.834141+0000","last_clean":"2024-09-21T10:03:53.834141+0000","last_became_active":"2024-09-21T10:02:46.659011+0000","last_became_peered":"2024-09-21T10:02:46.659011+0000","last_unstale":"2024-09-21T10:03:53.834141+0000","last_undegraded":"2024-09-21T10:03:53.834141+0000","last_fullsized":"2024-09-21T10:03:53.834141+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:04.650699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845276+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:03:53.845276+0000","last_peered":"2024-09-21T10:03:53.845276+0000","last_clean":"2024-09-21T10:03:53.845276+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:03:53.845276+0000","last_undegraded":"2024-09-21T10:03:53.845276+0000","last_fullsized":"2024-09-21T10:03:53.845276+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.844990+0000","last_change":"2024-09-21T10:03:53.844673+0000","last_active":"2024-09-21T10:03:53.844990+0000","last_peered":"2024-09-21T10:03:53.844990+0000","last_clean":"2024-09-21T10:03:53.844990+0000","last_became_active":"2024-09-21T10:03:53.844313+0000","last_became_peered":"2024-09-21T10:03:53.844313+0000","last_unstale":"2024-09-21T10:03:53.844990+0000","last_undegraded":"2024-09-21T10:03:53.844990+0000","last_fullsized":"2024-09-21T10:03:53.844990+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:42.529301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.841503+0000","last_change":"2024-09-21T10:03:53.841417+0000","last_active":"2024-09-21T10:03:53.841503+0000","last_peered":"2024-09-21T10:03:53.841503+0000","last_clean":"2024-09-21T10:03:53.841503+0000","last_became_active":"2024-09-21T10:03:53.840993+0000","last_became_peered":"2024-09-21T10:03:53.840993+0000","last_unstale":"2024-09-21T10:03:53.841503+0000","last_undegraded":"2024-09-21T10:03:53.841503+0000","last_fullsized":"2024-09-21T10:03:53.841503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:05.666490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.838352+0000","last_change":"2024-09-21T10:03:19.765682+0000","last_active":"2024-09-21T10:03:53.838352+0000","last_peered":"2024-09-21T10:03:53.838352+0000","last_clean":"2024-09-21T10:03:53.838352+0000","last_became_active":"2024-09-21T10:03:19.765344+0000","last_became_peered":"2024-09-21T10:03:19.765344+0000","last_unstale":"2024-09-21T10:03:53.838352+0000","last_undegraded":"2024-09-21T10:03:53.838352+0000","last_fullsized":"2024-09-21T10:03:53.838352+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:43.787280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839452+0000","last_change":"2024-09-21T10:03:53.839372+0000","last_active":"2024-09-21T10:03:53.839452+0000","last_peered":"2024-09-21T10:03:53.839452+0000","last_clean":"2024-09-21T10:03:53.839452+0000","last_became_active":"2024-09-21T10:03:53.839053+0000","last_became_peered":"2024-09-21T10:03:53.839053+0000","last_unstale":"2024-09-21T10:03:53.839452+0000","last_undegraded":"2024-09-21T10:03:53.839452+0000","last_fullsized":"2024-09-21T10:03:53.839452+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:29.647982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845364+0000","last_change":"2024-09-21T10:03:53.844946+0000","last_active":"2024-09-21T10:03:53.845364+0000","last_peered":"2024-09-21T10:03:53.845364+0000","last_clean":"2024-09-21T10:03:53.845364+0000","last_became_active":"2024-09-21T10:03:53.844613+0000","last_became_peered":"2024-09-21T10:03:53.844613+0000","last_unstale":"2024-09-21T10:03:53.845364+0000","last_undegraded":"2024-09-21T10:03:53.845364+0000","last_fullsized":"2024-09-21T10:03:53.845364+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:41.055573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.847034+0000","last_change":"2024-09-21T10:03:53.846948+0000","last_active":"2024-09-21T10:03:53.847034+0000","last_peered":"2024-09-21T10:03:53.847034+0000","last_clean":"2024-09-21T10:03:53.847034+0000","last_became_active":"2024-09-21T10:03:53.846614+0000","last_became_peered":"2024-09-21T10:03:53.846614+0000","last_unstale":"2024-09-21T10:03:53.847034+0000","last_undegraded":"2024-09-21T10:03:53.847034+0000","last_fullsized":"2024-09-21T10:03:53.847034+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:53.968372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845285+0000","last_change":"2024-09-21T10:03:53.845201+0000","last_active":"2024-09-21T10:03:53.845285+0000","last_peered":"2024-09-21T10:03:53.845285+0000","last_clean":"2024-09-21T10:03:53.845285+0000","last_became_active":"2024-09-21T10:03:53.843822+0000","last_became_peered":"2024-09-21T10:03:53.843822+0000","last_unstale":"2024-09-21T10:03:53.845285+0000","last_undegraded":"2024-09-21T10:03:53.845285+0000","last_fullsized":"2024-09-21T10:03:53.845285+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:28.895373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.828363+0000","last_change":"2024-09-21T10:03:19.768025+0000","last_active":"2024-09-21T10:03:52.828363+0000","last_peered":"2024-09-21T10:03:52.828363+0000","last_clean":"2024-09-21T10:03:52.828363+0000","last_became_active":"2024-09-21T10:03:19.767632+0000","last_became_peered":"2024-09-21T10:03:19.767632+0000","last_unstale":"2024-09-21T10:03:52.828363+0000","last_undegraded":"2024-09-21T10:03:52.828363+0000","last_fullsized":"2024-09-21T10:03:52.828363+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:40.023759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840358+0000","last_change":"2024-09-21T10:03:53.840263+0000","last_active":"2024-09-21T10:03:53.840358+0000","last_peered":"2024-09-21T10:03:53.840358+0000","last_clean":"2024-09-21T10:03:53.840358+0000","last_became_active":"2024-09-21T10:03:53.839894+0000","last_became_peered":"2024-09-21T10:03:53.839894+0000","last_unstale":"2024-09-21T10:03:53.840358+0000","last_undegraded":"2024-09-21T10:03:53.840358+0000","last_fullsized":"2024-09-21T10:03:53.840358+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:52.489580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830275+0000","last_change":"2024-09-21T10:02:12.846054+0000","last_active":"2024-09-21T10:03:53.830275+0000","last_peered":"2024-09-21T10:03:53.830275+0000","last_clean":"2024-09-21T10:03:53.830275+0000","last_became_active":"2024-09-21T10:02:12.843696+0000","last_became_peered":"2024-09-21T10:02:12.843696+0000","last_unstale":"2024-09-21T10:03:53.830275+0000","last_undegraded":"2024-09-21T10:03:53.830275+0000","last_fullsized":"2024-09-21T10:03:53.830275+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:32.280697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840992+0000","last_change":"2024-09-21T10:03:53.840992+0000","last_active":"2024-09-21T10:03:53.840992+0000","last_peered":"2024-09-21T10:03:53.840992+0000","last_clean":"2024-09-21T10:03:53.840992+0000","last_became_active":"2024-09-21T10:03:53.840551+0000","last_became_peered":"2024-09-21T10:03:53.840551+0000","last_unstale":"2024-09-21T10:03:53.840992+0000","last_undegraded":"2024-09-21T10:03:53.840992+0000","last_fullsized":"2024-09-21T10:03:53.840992+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:04.442126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836717+0000","last_change":"2024-09-21T10:02:46.659513+0000","last_active":"2024-09-21T10:03:52.836717+0000","last_peered":"2024-09-21T10:03:52.836717+0000","last_clean":"2024-09-21T10:03:52.836717+0000","last_became_active":"2024-09-21T10:02:46.659130+0000","last_became_peered":"2024-09-21T10:02:46.659130+0000","last_unstale":"2024-09-21T10:03:52.836717+0000","last_undegraded":"2024-09-21T10:03:52.836717+0000","last_fullsized":"2024-09-21T10:03:52.836717+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:24.792175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835006+0000","last_change":"2024-09-21T10:03:53.835006+0000","last_active":"2024-09-21T10:03:53.835006+0000","last_peered":"2024-09-21T10:03:53.835006+0000","last_clean":"2024-09-21T10:03:53.835006+0000","last_became_active":"2024-09-21T10:03:53.834608+0000","last_became_peered":"2024-09-21T10:03:53.834608+0000","last_unstale":"2024-09-21T10:03:53.835006+0000","last_undegraded":"2024-09-21T10:03:53.835006+0000","last_fullsized":"2024-09-21T10:03:53.835006+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:56.108606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":59,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.837866+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:03:52.837866+0000","last_peered":"2024-09-21T10:03:52.837866+0000","last_clean":"2024-09-21T10:03:52.837866+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:03:52.837866+0000","last_undegraded":"2024-09-21T10:03:52.837866+0000","last_fullsized":"2024-09-21T10:03:52.837866+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.844128+0000","last_change":"2024-09-21T10:03:53.844043+0000","last_active":"2024-09-21T10:03:53.844128+0000","last_peered":"2024-09-21T10:03:53.844128+0000","last_clean":"2024-09-21T10:03:53.844128+0000","last_became_active":"2024-09-21T10:03:53.842891+0000","last_became_peered":"2024-09-21T10:03:53.842891+0000","last_unstale":"2024-09-21T10:03:53.844128+0000","last_undegraded":"2024-09-21T10:03:53.844128+0000","last_fullsized":"2024-09-21T10:03:53.844128+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:39.891977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842964+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:03:53.842964+0000","last_peered":"2024-09-21T10:03:53.842964+0000","last_clean":"2024-09-21T10:03:53.842964+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:03:53.842964+0000","last_undegraded":"2024-09-21T10:03:53.842964+0000","last_fullsized":"2024-09-21T10:03:53.842964+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.831425+0000","last_change":"2024-09-21T10:02:46.659818+0000","last_active":"2024-09-21T10:03:53.831425+0000","last_peered":"2024-09-21T10:03:53.831425+0000","last_clean":"2024-09-21T10:03:53.831425+0000","last_became_active":"2024-09-21T10:02:46.659457+0000","last_became_peered":"2024-09-21T10:02:46.659457+0000","last_unstale":"2024-09-21T10:03:53.831425+0000","last_undegraded":"2024-09-21T10:03:53.831425+0000","last_fullsized":"2024-09-21T10:03:53.831425+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:21.962330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830391+0000","last_change":"2024-09-21T10:03:19.769549+0000","last_active":"2024-09-21T10:03:53.830391+0000","last_peered":"2024-09-21T10:03:53.830391+0000","last_clean":"2024-09-21T10:03:53.830391+0000","last_became_active":"2024-09-21T10:03:19.768854+0000","last_became_peered":"2024-09-21T10:03:19.768854+0000","last_unstale":"2024-09-21T10:03:53.830391+0000","last_undegraded":"2024-09-21T10:03:53.830391+0000","last_fullsized":"2024-09-21T10:03:53.830391+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:29.415039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234183,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37008,"kb_used_data":5884,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708144,"statfs":{"total":95995035648,"available":95957139456,"internally_reserved":0,"allocated":6025216,"data_stored":5417204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495821,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32908,"kb_used_data":1784,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712244,"statfs":{"total":95995035648,"available":95961337856,"internally_reserved":0,"allocated":1826816,"data_stored":1223104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757458,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28216,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716936,"statfs":{"total":95995035648,"available":95966142464,"internally_reserved":0,"allocated":1224704,"data_stored":628562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019094,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32332,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712820,"statfs":{"total":95995035648,"available":95961927680,"internally_reserved":0,"allocated":1236992,"data_stored":632793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435278,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32888,"kb_used_data":1188,"kb_used_omap":5,"kb_used_meta":31674,"kb_avail":93712264,"statfs":{"total":95995035648,"available":95961358336,"internally_reserved":0,"allocated":1216512,"data_stored":627993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":32434216},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598805,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1188,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1216512,"data_stored":628211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":28172749},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762331,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36976,"kb_used_data":5280,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708176,"statfs":{"total":95995035648,"available":95957172224,"internally_reserved":0,"allocated":5406720,"data_stored":4821953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":32432381},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925858,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33404,"kb_used_data":5868,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711748,"statfs":{"total":95995035648,"available":95960829952,"internally_reserved":0,"allocated":6008832,"data_stored":5413179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":28172478},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:04:55.931 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:04:56.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:56 smithi032 ceph-mon[105748]: from='client.34557 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:56 smithi032 ceph-mon[101299]: from='client.34557 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:56.272 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:56.488 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:56 smithi045 ceph-mon[95976]: from='client.34557 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:57.370 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:57 smithi032 ceph-mon[101299]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:57.371 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:57 smithi032 ceph-mon[105748]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:57.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:57 smithi045 ceph-mon[95976]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:58.069 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:04:58.070 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:04:58.814 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:58 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:58.814 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:58 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:58.814 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:58 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:58.814 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:58 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:58.840 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":282,"stamp":"2024-09-21T10:04:56.124669+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":262452,"kb_used_data":23596,"kb_used_omap":35,"kb_used_meta":238684,"kb_avail":749698764,"statfs":{"total":767960285184,"available":767691534336,"internally_reserved":0,"allocated":24162304,"data_stored":19392999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":244413134},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002851"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843669+0000","last_change":"2024-09-21T10:03:53.843548+0000","last_active":"2024-09-21T10:03:53.843669+0000","last_peered":"2024-09-21T10:03:53.843669+0000","last_clean":"2024-09-21T10:03:53.843669+0000","last_became_active":"2024-09-21T10:03:53.843182+0000","last_became_peered":"2024-09-21T10:03:53.843182+0000","last_unstale":"2024-09-21T10:03:53.843669+0000","last_undegraded":"2024-09-21T10:03:53.843669+0000","last_fullsized":"2024-09-21T10:03:53.843669+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:09.464209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834897+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:03:53.834897+0000","last_peered":"2024-09-21T10:03:53.834897+0000","last_clean":"2024-09-21T10:03:53.834897+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:03:53.834897+0000","last_undegraded":"2024-09-21T10:03:53.834897+0000","last_fullsized":"2024-09-21T10:03:53.834897+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840530+0000","last_change":"2024-09-21T10:03:53.839987+0000","last_active":"2024-09-21T10:03:53.840530+0000","last_peered":"2024-09-21T10:03:53.840530+0000","last_clean":"2024-09-21T10:03:53.840530+0000","last_became_active":"2024-09-21T10:03:53.839664+0000","last_became_peered":"2024-09-21T10:03:53.839664+0000","last_unstale":"2024-09-21T10:03:53.840530+0000","last_undegraded":"2024-09-21T10:03:53.840530+0000","last_fullsized":"2024-09-21T10:03:53.840530+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:51.726392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837862+0000","last_change":"2024-09-21T10:03:53.837496+0000","last_active":"2024-09-21T10:03:53.837862+0000","last_peered":"2024-09-21T10:03:53.837862+0000","last_clean":"2024-09-21T10:03:53.837862+0000","last_became_active":"2024-09-21T10:03:53.837166+0000","last_became_peered":"2024-09-21T10:03:53.837166+0000","last_unstale":"2024-09-21T10:03:53.837862+0000","last_undegraded":"2024-09-21T10:03:53.837862+0000","last_fullsized":"2024-09-21T10:03:53.837862+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:45.795571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845452+0000","last_change":"2024-09-21T10:03:53.844990+0000","last_active":"2024-09-21T10:03:53.845452+0000","last_peered":"2024-09-21T10:03:53.845452+0000","last_clean":"2024-09-21T10:03:53.845452+0000","last_became_active":"2024-09-21T10:03:53.844664+0000","last_became_peered":"2024-09-21T10:03:53.844664+0000","last_unstale":"2024-09-21T10:03:53.845452+0000","last_undegraded":"2024-09-21T10:03:53.845452+0000","last_fullsized":"2024-09-21T10:03:53.845452+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:01.031522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835247+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:03:53.835247+0000","last_peered":"2024-09-21T10:03:53.835247+0000","last_clean":"2024-09-21T10:03:53.835247+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:03:53.835247+0000","last_undegraded":"2024-09-21T10:03:53.835247+0000","last_fullsized":"2024-09-21T10:03:53.835247+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845872+0000","last_change":"2024-09-21T10:03:53.845785+0000","last_active":"2024-09-21T10:03:53.845872+0000","last_peered":"2024-09-21T10:03:53.845872+0000","last_clean":"2024-09-21T10:03:53.845872+0000","last_became_active":"2024-09-21T10:03:53.845303+0000","last_became_peered":"2024-09-21T10:03:53.845303+0000","last_unstale":"2024-09-21T10:03:53.845872+0000","last_undegraded":"2024-09-21T10:03:53.845872+0000","last_fullsized":"2024-09-21T10:03:53.845872+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:04.976007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837943+0000","last_change":"2024-09-21T10:02:12.838786+0000","last_active":"2024-09-21T10:03:53.837943+0000","last_peered":"2024-09-21T10:03:53.837943+0000","last_clean":"2024-09-21T10:03:53.837943+0000","last_became_active":"2024-09-21T10:02:12.838242+0000","last_became_peered":"2024-09-21T10:02:12.838242+0000","last_unstale":"2024-09-21T10:03:53.837943+0000","last_undegraded":"2024-09-21T10:03:53.837943+0000","last_fullsized":"2024-09-21T10:03:53.837943+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:31.071136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835170+0000","last_change":"2024-09-21T10:02:12.840334+0000","last_active":"2024-09-21T10:03:53.835170+0000","last_peered":"2024-09-21T10:03:53.835170+0000","last_clean":"2024-09-21T10:03:53.835170+0000","last_became_active":"2024-09-21T10:02:12.839928+0000","last_became_peered":"2024-09-21T10:02:12.839928+0000","last_unstale":"2024-09-21T10:03:53.835170+0000","last_undegraded":"2024-09-21T10:03:53.835170+0000","last_fullsized":"2024-09-21T10:03:53.835170+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:38.593644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836136+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T10:03:52.836136+0000","last_peered":"2024-09-21T10:03:52.836136+0000","last_clean":"2024-09-21T10:03:52.836136+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T10:03:52.836136+0000","last_undegraded":"2024-09-21T10:03:52.836136+0000","last_fullsized":"2024-09-21T10:03:52.836136+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.836116+0000","last_change":"2024-09-21T10:02:46.662169+0000","last_active":"2024-09-21T10:03:53.836116+0000","last_peered":"2024-09-21T10:03:53.836116+0000","last_clean":"2024-09-21T10:03:53.836116+0000","last_became_active":"2024-09-21T10:02:46.661721+0000","last_became_peered":"2024-09-21T10:02:46.661721+0000","last_unstale":"2024-09-21T10:03:53.836116+0000","last_undegraded":"2024-09-21T10:03:53.836116+0000","last_fullsized":"2024-09-21T10:03:53.836116+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:10.414152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840738+0000","last_change":"2024-09-21T10:03:53.840738+0000","last_active":"2024-09-21T10:03:53.840738+0000","last_peered":"2024-09-21T10:03:53.840738+0000","last_clean":"2024-09-21T10:03:53.840738+0000","last_became_active":"2024-09-21T10:03:53.840363+0000","last_became_peered":"2024-09-21T10:03:53.840363+0000","last_unstale":"2024-09-21T10:03:53.840738+0000","last_undegraded":"2024-09-21T10:03:53.840738+0000","last_fullsized":"2024-09-21T10:03:53.840738+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:26.371181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835739+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T10:03:53.835739+0000","last_peered":"2024-09-21T10:03:53.835739+0000","last_clean":"2024-09-21T10:03:53.835739+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T10:03:53.835739+0000","last_undegraded":"2024-09-21T10:03:53.835739+0000","last_fullsized":"2024-09-21T10:03:53.835739+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833635+0000","last_change":"2024-09-21T10:02:46.663127+0000","last_active":"2024-09-21T10:03:52.833635+0000","last_peered":"2024-09-21T10:03:52.833635+0000","last_clean":"2024-09-21T10:03:52.833635+0000","last_became_active":"2024-09-21T10:02:46.662791+0000","last_became_peered":"2024-09-21T10:02:46.662791+0000","last_unstale":"2024-09-21T10:03:52.833635+0000","last_undegraded":"2024-09-21T10:03:52.833635+0000","last_fullsized":"2024-09-21T10:03:52.833635+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:59.990122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843912+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:03:53.843912+0000","last_peered":"2024-09-21T10:03:53.843912+0000","last_clean":"2024-09-21T10:03:53.843912+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:03:53.843912+0000","last_undegraded":"2024-09-21T10:03:53.843912+0000","last_fullsized":"2024-09-21T10:03:53.843912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830367+0000","last_change":"2024-09-21T10:02:46.662651+0000","last_active":"2024-09-21T10:03:53.830367+0000","last_peered":"2024-09-21T10:03:53.830367+0000","last_clean":"2024-09-21T10:03:53.830367+0000","last_became_active":"2024-09-21T10:02:46.662304+0000","last_became_peered":"2024-09-21T10:02:46.662304+0000","last_unstale":"2024-09-21T10:03:53.830367+0000","last_undegraded":"2024-09-21T10:03:53.830367+0000","last_fullsized":"2024-09-21T10:03:53.830367+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:06.746929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840595+0000","last_change":"2024-09-21T10:03:53.840511+0000","last_active":"2024-09-21T10:03:53.840595+0000","last_peered":"2024-09-21T10:03:53.840595+0000","last_clean":"2024-09-21T10:03:53.840595+0000","last_became_active":"2024-09-21T10:03:53.840136+0000","last_became_peered":"2024-09-21T10:03:53.840136+0000","last_unstale":"2024-09-21T10:03:53.840595+0000","last_undegraded":"2024-09-21T10:03:53.840595+0000","last_fullsized":"2024-09-21T10:03:53.840595+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:24.765990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840155+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:03:53.840155+0000","last_peered":"2024-09-21T10:03:53.840155+0000","last_clean":"2024-09-21T10:03:53.840155+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:03:53.840155+0000","last_undegraded":"2024-09-21T10:03:53.840155+0000","last_fullsized":"2024-09-21T10:03:53.840155+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833841+0000","last_change":"2024-09-21T10:02:46.663900+0000","last_active":"2024-09-21T10:03:52.833841+0000","last_peered":"2024-09-21T10:03:52.833841+0000","last_clean":"2024-09-21T10:03:52.833841+0000","last_became_active":"2024-09-21T10:02:46.663594+0000","last_became_peered":"2024-09-21T10:02:46.663594+0000","last_unstale":"2024-09-21T10:03:52.833841+0000","last_undegraded":"2024-09-21T10:03:52.833841+0000","last_fullsized":"2024-09-21T10:03:52.833841+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:24.140061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832354+0000","last_change":"2024-09-21T10:03:19.765623+0000","last_active":"2024-09-21T10:03:53.832354+0000","last_peered":"2024-09-21T10:03:53.832354+0000","last_clean":"2024-09-21T10:03:53.832354+0000","last_became_active":"2024-09-21T10:03:19.765227+0000","last_became_peered":"2024-09-21T10:03:19.765227+0000","last_unstale":"2024-09-21T10:03:53.832354+0000","last_undegraded":"2024-09-21T10:03:53.832354+0000","last_fullsized":"2024-09-21T10:03:53.832354+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:52.227826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.829335+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:03:52.829335+0000","last_peered":"2024-09-21T10:03:52.829335+0000","last_clean":"2024-09-21T10:03:52.829335+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:03:52.829335+0000","last_undegraded":"2024-09-21T10:03:52.829335+0000","last_fullsized":"2024-09-21T10:03:52.829335+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832211+0000","last_change":"2024-09-21T10:02:46.664014+0000","last_active":"2024-09-21T10:03:53.832211+0000","last_peered":"2024-09-21T10:03:53.832211+0000","last_clean":"2024-09-21T10:03:53.832211+0000","last_became_active":"2024-09-21T10:02:46.663640+0000","last_became_peered":"2024-09-21T10:02:46.663640+0000","last_unstale":"2024-09-21T10:03:53.832211+0000","last_undegraded":"2024-09-21T10:03:53.832211+0000","last_fullsized":"2024-09-21T10:03:53.832211+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:03.733462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845713+0000","last_change":"2024-09-21T10:03:53.845620+0000","last_active":"2024-09-21T10:03:53.845713+0000","last_peered":"2024-09-21T10:03:53.845713+0000","last_clean":"2024-09-21T10:03:53.845713+0000","last_became_active":"2024-09-21T10:03:53.845290+0000","last_became_peered":"2024-09-21T10:03:53.845290+0000","last_unstale":"2024-09-21T10:03:53.845713+0000","last_undegraded":"2024-09-21T10:03:53.845713+0000","last_fullsized":"2024-09-21T10:03:53.845713+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:20.076623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.846700+0000","last_change":"2024-09-21T10:03:53.846606+0000","last_active":"2024-09-21T10:03:53.846700+0000","last_peered":"2024-09-21T10:03:53.846700+0000","last_clean":"2024-09-21T10:03:53.846700+0000","last_became_active":"2024-09-21T10:03:53.846245+0000","last_became_peered":"2024-09-21T10:03:53.846245+0000","last_unstale":"2024-09-21T10:03:53.846700+0000","last_undegraded":"2024-09-21T10:03:53.846700+0000","last_fullsized":"2024-09-21T10:03:53.846700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:20.745470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":405,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835020+0000","last_change":"2024-09-21T10:02:12.840225+0000","last_active":"2024-09-21T10:03:53.835020+0000","last_peered":"2024-09-21T10:03:53.835020+0000","last_clean":"2024-09-21T10:03:53.835020+0000","last_became_active":"2024-09-21T10:02:12.839813+0000","last_became_peered":"2024-09-21T10:02:12.839813+0000","last_unstale":"2024-09-21T10:03:53.835020+0000","last_undegraded":"2024-09-21T10:03:53.835020+0000","last_fullsized":"2024-09-21T10:03:53.835020+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:36.630881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":423,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836626+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:03:52.836626+0000","last_peered":"2024-09-21T10:03:52.836626+0000","last_clean":"2024-09-21T10:03:52.836626+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:03:52.836626+0000","last_undegraded":"2024-09-21T10:03:52.836626+0000","last_fullsized":"2024-09-21T10:03:52.836626+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843425+0000","last_change":"2024-09-21T10:03:53.843344+0000","last_active":"2024-09-21T10:03:53.843425+0000","last_peered":"2024-09-21T10:03:53.843425+0000","last_clean":"2024-09-21T10:03:53.843425+0000","last_became_active":"2024-09-21T10:03:53.842304+0000","last_became_peered":"2024-09-21T10:03:53.842304+0000","last_unstale":"2024-09-21T10:03:53.843425+0000","last_undegraded":"2024-09-21T10:03:53.843425+0000","last_fullsized":"2024-09-21T10:03:53.843425+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:31.221735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.833226+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:03:52.833226+0000","last_peered":"2024-09-21T10:03:52.833226+0000","last_clean":"2024-09-21T10:03:52.833226+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:03:52.833226+0000","last_undegraded":"2024-09-21T10:03:52.833226+0000","last_fullsized":"2024-09-21T10:03:52.833226+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":568,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.832252+0000","last_change":"2024-09-21T10:02:46.663394+0000","last_active":"2024-09-21T10:03:52.832252+0000","last_peered":"2024-09-21T10:03:52.832252+0000","last_clean":"2024-09-21T10:03:52.832252+0000","last_became_active":"2024-09-21T10:02:46.662879+0000","last_became_peered":"2024-09-21T10:02:46.662879+0000","last_unstale":"2024-09-21T10:03:52.832252+0000","last_undegraded":"2024-09-21T10:03:52.832252+0000","last_fullsized":"2024-09-21T10:03:52.832252+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.125795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835797+0000","last_change":"2024-09-21T10:03:53.835329+0000","last_active":"2024-09-21T10:03:53.835797+0000","last_peered":"2024-09-21T10:03:53.835797+0000","last_clean":"2024-09-21T10:03:53.835797+0000","last_became_active":"2024-09-21T10:03:53.834934+0000","last_became_peered":"2024-09-21T10:03:53.834934+0000","last_unstale":"2024-09-21T10:03:53.835797+0000","last_undegraded":"2024-09-21T10:03:53.835797+0000","last_fullsized":"2024-09-21T10:03:53.835797+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:38.355817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830913+0000","last_change":"2024-09-21T10:03:19.770016+0000","last_active":"2024-09-21T10:03:53.830913+0000","last_peered":"2024-09-21T10:03:53.830913+0000","last_clean":"2024-09-21T10:03:53.830913+0000","last_became_active":"2024-09-21T10:03:19.768866+0000","last_became_peered":"2024-09-21T10:03:19.768866+0000","last_unstale":"2024-09-21T10:03:53.830913+0000","last_undegraded":"2024-09-21T10:03:53.830913+0000","last_fullsized":"2024-09-21T10:03:53.830913+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:25.068940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":573,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840408+0000","last_change":"2024-09-21T10:03:53.840322+0000","last_active":"2024-09-21T10:03:53.840408+0000","last_peered":"2024-09-21T10:03:53.840408+0000","last_clean":"2024-09-21T10:03:53.840408+0000","last_became_active":"2024-09-21T10:03:53.839753+0000","last_became_peered":"2024-09-21T10:03:53.839753+0000","last_unstale":"2024-09-21T10:03:53.840408+0000","last_undegraded":"2024-09-21T10:03:53.840408+0000","last_fullsized":"2024-09-21T10:03:53.840408+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:26.364888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835436+0000","last_change":"2024-09-21T10:03:19.765402+0000","last_active":"2024-09-21T10:03:53.835436+0000","last_peered":"2024-09-21T10:03:53.835436+0000","last_clean":"2024-09-21T10:03:53.835436+0000","last_became_active":"2024-09-21T10:03:19.765036+0000","last_became_peered":"2024-09-21T10:03:19.765036+0000","last_unstale":"2024-09-21T10:03:53.835436+0000","last_undegraded":"2024-09-21T10:03:53.835436+0000","last_fullsized":"2024-09-21T10:03:53.835436+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:32.230377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837936+0000","last_change":"2024-09-21T10:02:46.660814+0000","last_active":"2024-09-21T10:03:53.837936+0000","last_peered":"2024-09-21T10:03:53.837936+0000","last_clean":"2024-09-21T10:03:53.837936+0000","last_became_active":"2024-09-21T10:02:46.660325+0000","last_became_peered":"2024-09-21T10:02:46.660325+0000","last_unstale":"2024-09-21T10:03:53.837936+0000","last_undegraded":"2024-09-21T10:03:53.837936+0000","last_fullsized":"2024-09-21T10:03:53.837936+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:44.427243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":811,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836665+0000","last_change":"2024-09-21T10:02:46.657107+0000","last_active":"2024-09-21T10:03:52.836665+0000","last_peered":"2024-09-21T10:03:52.836665+0000","last_clean":"2024-09-21T10:03:52.836665+0000","last_became_active":"2024-09-21T10:02:46.656633+0000","last_became_peered":"2024-09-21T10:02:46.656633+0000","last_unstale":"2024-09-21T10:03:52.836665+0000","last_undegraded":"2024-09-21T10:03:52.836665+0000","last_fullsized":"2024-09-21T10:03:52.836665+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:42:18.534832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.830018+0000","last_change":"2024-09-21T10:02:12.840581+0000","last_active":"2024-09-21T10:03:52.830018+0000","last_peered":"2024-09-21T10:03:52.830018+0000","last_clean":"2024-09-21T10:03:52.830018+0000","last_became_active":"2024-09-21T10:02:12.840033+0000","last_became_peered":"2024-09-21T10:02:12.840033+0000","last_unstale":"2024-09-21T10:03:52.830018+0000","last_undegraded":"2024-09-21T10:03:52.830018+0000","last_fullsized":"2024-09-21T10:03:52.830018+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:22.449586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843937+0000","last_change":"2024-09-21T10:03:53.843551+0000","last_active":"2024-09-21T10:03:53.843937+0000","last_peered":"2024-09-21T10:03:53.843937+0000","last_clean":"2024-09-21T10:03:53.843937+0000","last_became_active":"2024-09-21T10:03:53.843007+0000","last_became_peered":"2024-09-21T10:03:53.843007+0000","last_unstale":"2024-09-21T10:03:53.843937+0000","last_undegraded":"2024-09-21T10:03:53.843937+0000","last_fullsized":"2024-09-21T10:03:53.843937+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:11.757447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.843567+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:03:53.843567+0000","last_peered":"2024-09-21T10:03:53.843567+0000","last_clean":"2024-09-21T10:03:53.843567+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:03:53.843567+0000","last_undegraded":"2024-09-21T10:03:53.843567+0000","last_fullsized":"2024-09-21T10:03:53.843567+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":530,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.836990+0000","last_change":"2024-09-21T10:02:46.662410+0000","last_active":"2024-09-21T10:03:53.836990+0000","last_peered":"2024-09-21T10:03:53.836990+0000","last_clean":"2024-09-21T10:03:53.836990+0000","last_became_active":"2024-09-21T10:02:46.661506+0000","last_became_peered":"2024-09-21T10:02:46.661506+0000","last_unstale":"2024-09-21T10:03:53.836990+0000","last_undegraded":"2024-09-21T10:03:53.836990+0000","last_fullsized":"2024-09-21T10:03:53.836990+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:40:33.566844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834945+0000","last_change":"2024-09-21T10:02:46.665713+0000","last_active":"2024-09-21T10:03:53.834945+0000","last_peered":"2024-09-21T10:03:53.834945+0000","last_clean":"2024-09-21T10:03:53.834945+0000","last_became_active":"2024-09-21T10:02:46.665316+0000","last_became_peered":"2024-09-21T10:02:46.665316+0000","last_unstale":"2024-09-21T10:03:53.834945+0000","last_undegraded":"2024-09-21T10:03:53.834945+0000","last_fullsized":"2024-09-21T10:03:53.834945+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.951202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.837129+0000","last_change":"2024-09-21T10:03:19.768187+0000","last_active":"2024-09-21T10:03:53.837129+0000","last_peered":"2024-09-21T10:03:53.837129+0000","last_clean":"2024-09-21T10:03:53.837129+0000","last_became_active":"2024-09-21T10:03:19.767801+0000","last_became_peered":"2024-09-21T10:03:19.767801+0000","last_unstale":"2024-09-21T10:03:53.837129+0000","last_undegraded":"2024-09-21T10:03:53.837129+0000","last_fullsized":"2024-09-21T10:03:53.837129+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:38.042481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":511,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835831+0000","last_change":"2024-09-21T10:02:12.840296+0000","last_active":"2024-09-21T10:03:53.835831+0000","last_peered":"2024-09-21T10:03:53.835831+0000","last_clean":"2024-09-21T10:03:53.835831+0000","last_became_active":"2024-09-21T10:02:12.839881+0000","last_became_peered":"2024-09-21T10:02:12.839881+0000","last_unstale":"2024-09-21T10:03:53.835831+0000","last_undegraded":"2024-09-21T10:03:53.835831+0000","last_fullsized":"2024-09-21T10:03:53.835831+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:49.166865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830265+0000","last_change":"2024-09-21T10:02:12.845158+0000","last_active":"2024-09-21T10:03:53.830265+0000","last_peered":"2024-09-21T10:03:53.830265+0000","last_clean":"2024-09-21T10:03:53.830265+0000","last_became_active":"2024-09-21T10:02:12.843399+0000","last_became_peered":"2024-09-21T10:02:12.843399+0000","last_unstale":"2024-09-21T10:03:53.830265+0000","last_undegraded":"2024-09-21T10:03:53.830265+0000","last_fullsized":"2024-09-21T10:03:53.830265+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:27.552491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842700+0000","last_change":"2024-09-21T10:03:53.842610+0000","last_active":"2024-09-21T10:03:53.842700+0000","last_peered":"2024-09-21T10:03:53.842700+0000","last_clean":"2024-09-21T10:03:53.842700+0000","last_became_active":"2024-09-21T10:03:53.841888+0000","last_became_peered":"2024-09-21T10:03:53.841888+0000","last_unstale":"2024-09-21T10:03:53.842700+0000","last_undegraded":"2024-09-21T10:03:53.842700+0000","last_fullsized":"2024-09-21T10:03:53.842700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:05.833771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":636,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835309+0000","last_change":"2024-09-21T10:02:46.665711+0000","last_active":"2024-09-21T10:03:53.835309+0000","last_peered":"2024-09-21T10:03:53.835309+0000","last_clean":"2024-09-21T10:03:53.835309+0000","last_became_active":"2024-09-21T10:02:46.665165+0000","last_became_peered":"2024-09-21T10:02:46.665165+0000","last_unstale":"2024-09-21T10:03:53.835309+0000","last_undegraded":"2024-09-21T10:03:53.835309+0000","last_fullsized":"2024-09-21T10:03:53.835309+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:08.886554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839031+0000","last_change":"2024-09-21T10:03:53.838941+0000","last_active":"2024-09-21T10:03:53.839031+0000","last_peered":"2024-09-21T10:03:53.839031+0000","last_clean":"2024-09-21T10:03:53.839031+0000","last_became_active":"2024-09-21T10:03:53.838646+0000","last_became_peered":"2024-09-21T10:03:53.838646+0000","last_unstale":"2024-09-21T10:03:53.839031+0000","last_undegraded":"2024-09-21T10:03:53.839031+0000","last_fullsized":"2024-09-21T10:03:53.839031+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:25.167057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830074+0000","last_change":"2024-09-21T10:02:46.661363+0000","last_active":"2024-09-21T10:03:53.830074+0000","last_peered":"2024-09-21T10:03:53.830074+0000","last_clean":"2024-09-21T10:03:53.830074+0000","last_became_active":"2024-09-21T10:02:46.661005+0000","last_became_peered":"2024-09-21T10:02:46.661005+0000","last_unstale":"2024-09-21T10:03:53.830074+0000","last_undegraded":"2024-09-21T10:03:53.830074+0000","last_fullsized":"2024-09-21T10:03:53.830074+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:42.908471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":349,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.838167+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:03:53.838167+0000","last_peered":"2024-09-21T10:03:53.838167+0000","last_clean":"2024-09-21T10:03:53.838167+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:03:53.838167+0000","last_undegraded":"2024-09-21T10:03:53.838167+0000","last_fullsized":"2024-09-21T10:03:53.838167+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.833021+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:03:53.833021+0000","last_peered":"2024-09-21T10:03:53.833021+0000","last_clean":"2024-09-21T10:03:53.833021+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:03:53.833021+0000","last_undegraded":"2024-09-21T10:03:53.833021+0000","last_fullsized":"2024-09-21T10:03:53.833021+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832815+0000","last_change":"2024-09-21T10:02:46.667831+0000","last_active":"2024-09-21T10:03:53.832815+0000","last_peered":"2024-09-21T10:03:53.832815+0000","last_clean":"2024-09-21T10:03:53.832815+0000","last_became_active":"2024-09-21T10:02:46.667507+0000","last_became_peered":"2024-09-21T10:02:46.667507+0000","last_unstale":"2024-09-21T10:03:53.832815+0000","last_undegraded":"2024-09-21T10:03:53.832815+0000","last_fullsized":"2024-09-21T10:03:53.832815+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:23.990802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.832211+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:03:53.832211+0000","last_peered":"2024-09-21T10:03:53.832211+0000","last_clean":"2024-09-21T10:03:53.832211+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:03:53.832211+0000","last_undegraded":"2024-09-21T10:03:53.832211+0000","last_fullsized":"2024-09-21T10:03:53.832211+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836450+0000","last_change":"2024-09-21T10:03:19.767147+0000","last_active":"2024-09-21T10:03:52.836450+0000","last_peered":"2024-09-21T10:03:52.836450+0000","last_clean":"2024-09-21T10:03:52.836450+0000","last_became_active":"2024-09-21T10:03:19.766822+0000","last_became_peered":"2024-09-21T10:03:19.766822+0000","last_unstale":"2024-09-21T10:03:52.836450+0000","last_undegraded":"2024-09-21T10:03:52.836450+0000","last_fullsized":"2024-09-21T10:03:52.836450+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:38.875912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842074+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:03:53.842074+0000","last_peered":"2024-09-21T10:03:53.842074+0000","last_clean":"2024-09-21T10:03:53.842074+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:03:53.842074+0000","last_undegraded":"2024-09-21T10:03:53.842074+0000","last_fullsized":"2024-09-21T10:03:53.842074+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.834141+0000","last_change":"2024-09-21T10:02:46.659383+0000","last_active":"2024-09-21T10:03:53.834141+0000","last_peered":"2024-09-21T10:03:53.834141+0000","last_clean":"2024-09-21T10:03:53.834141+0000","last_became_active":"2024-09-21T10:02:46.659011+0000","last_became_peered":"2024-09-21T10:02:46.659011+0000","last_unstale":"2024-09-21T10:03:53.834141+0000","last_undegraded":"2024-09-21T10:03:53.834141+0000","last_fullsized":"2024-09-21T10:03:53.834141+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:04.650699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845276+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:03:53.845276+0000","last_peered":"2024-09-21T10:03:53.845276+0000","last_clean":"2024-09-21T10:03:53.845276+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:03:53.845276+0000","last_undegraded":"2024-09-21T10:03:53.845276+0000","last_fullsized":"2024-09-21T10:03:53.845276+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.844990+0000","last_change":"2024-09-21T10:03:53.844673+0000","last_active":"2024-09-21T10:03:53.844990+0000","last_peered":"2024-09-21T10:03:53.844990+0000","last_clean":"2024-09-21T10:03:53.844990+0000","last_became_active":"2024-09-21T10:03:53.844313+0000","last_became_peered":"2024-09-21T10:03:53.844313+0000","last_unstale":"2024-09-21T10:03:53.844990+0000","last_undegraded":"2024-09-21T10:03:53.844990+0000","last_fullsized":"2024-09-21T10:03:53.844990+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:42.529301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.841503+0000","last_change":"2024-09-21T10:03:53.841417+0000","last_active":"2024-09-21T10:03:53.841503+0000","last_peered":"2024-09-21T10:03:53.841503+0000","last_clean":"2024-09-21T10:03:53.841503+0000","last_became_active":"2024-09-21T10:03:53.840993+0000","last_became_peered":"2024-09-21T10:03:53.840993+0000","last_unstale":"2024-09-21T10:03:53.841503+0000","last_undegraded":"2024-09-21T10:03:53.841503+0000","last_fullsized":"2024-09-21T10:03:53.841503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:05.666490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.838352+0000","last_change":"2024-09-21T10:03:19.765682+0000","last_active":"2024-09-21T10:03:53.838352+0000","last_peered":"2024-09-21T10:03:53.838352+0000","last_clean":"2024-09-21T10:03:53.838352+0000","last_became_active":"2024-09-21T10:03:19.765344+0000","last_became_peered":"2024-09-21T10:03:19.765344+0000","last_unstale":"2024-09-21T10:03:53.838352+0000","last_undegraded":"2024-09-21T10:03:53.838352+0000","last_fullsized":"2024-09-21T10:03:53.838352+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:43.787280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.839452+0000","last_change":"2024-09-21T10:03:53.839372+0000","last_active":"2024-09-21T10:03:53.839452+0000","last_peered":"2024-09-21T10:03:53.839452+0000","last_clean":"2024-09-21T10:03:53.839452+0000","last_became_active":"2024-09-21T10:03:53.839053+0000","last_became_peered":"2024-09-21T10:03:53.839053+0000","last_unstale":"2024-09-21T10:03:53.839452+0000","last_undegraded":"2024-09-21T10:03:53.839452+0000","last_fullsized":"2024-09-21T10:03:53.839452+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:29.647982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845364+0000","last_change":"2024-09-21T10:03:53.844946+0000","last_active":"2024-09-21T10:03:53.845364+0000","last_peered":"2024-09-21T10:03:53.845364+0000","last_clean":"2024-09-21T10:03:53.845364+0000","last_became_active":"2024-09-21T10:03:53.844613+0000","last_became_peered":"2024-09-21T10:03:53.844613+0000","last_unstale":"2024-09-21T10:03:53.845364+0000","last_undegraded":"2024-09-21T10:03:53.845364+0000","last_fullsized":"2024-09-21T10:03:53.845364+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:41.055573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.847034+0000","last_change":"2024-09-21T10:03:53.846948+0000","last_active":"2024-09-21T10:03:53.847034+0000","last_peered":"2024-09-21T10:03:53.847034+0000","last_clean":"2024-09-21T10:03:53.847034+0000","last_became_active":"2024-09-21T10:03:53.846614+0000","last_became_peered":"2024-09-21T10:03:53.846614+0000","last_unstale":"2024-09-21T10:03:53.847034+0000","last_undegraded":"2024-09-21T10:03:53.847034+0000","last_fullsized":"2024-09-21T10:03:53.847034+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:53.968372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.845285+0000","last_change":"2024-09-21T10:03:53.845201+0000","last_active":"2024-09-21T10:03:53.845285+0000","last_peered":"2024-09-21T10:03:53.845285+0000","last_clean":"2024-09-21T10:03:53.845285+0000","last_became_active":"2024-09-21T10:03:53.843822+0000","last_became_peered":"2024-09-21T10:03:53.843822+0000","last_unstale":"2024-09-21T10:03:53.845285+0000","last_undegraded":"2024-09-21T10:03:53.845285+0000","last_fullsized":"2024-09-21T10:03:53.845285+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:28.895373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.828363+0000","last_change":"2024-09-21T10:03:19.768025+0000","last_active":"2024-09-21T10:03:52.828363+0000","last_peered":"2024-09-21T10:03:52.828363+0000","last_clean":"2024-09-21T10:03:52.828363+0000","last_became_active":"2024-09-21T10:03:19.767632+0000","last_became_peered":"2024-09-21T10:03:19.767632+0000","last_unstale":"2024-09-21T10:03:52.828363+0000","last_undegraded":"2024-09-21T10:03:52.828363+0000","last_fullsized":"2024-09-21T10:03:52.828363+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:40.023759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840358+0000","last_change":"2024-09-21T10:03:53.840263+0000","last_active":"2024-09-21T10:03:53.840358+0000","last_peered":"2024-09-21T10:03:53.840358+0000","last_clean":"2024-09-21T10:03:53.840358+0000","last_became_active":"2024-09-21T10:03:53.839894+0000","last_became_peered":"2024-09-21T10:03:53.839894+0000","last_unstale":"2024-09-21T10:03:53.840358+0000","last_undegraded":"2024-09-21T10:03:53.840358+0000","last_fullsized":"2024-09-21T10:03:53.840358+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:52.489580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830275+0000","last_change":"2024-09-21T10:02:12.846054+0000","last_active":"2024-09-21T10:03:53.830275+0000","last_peered":"2024-09-21T10:03:53.830275+0000","last_clean":"2024-09-21T10:03:53.830275+0000","last_became_active":"2024-09-21T10:02:12.843696+0000","last_became_peered":"2024-09-21T10:02:12.843696+0000","last_unstale":"2024-09-21T10:03:53.830275+0000","last_undegraded":"2024-09-21T10:03:53.830275+0000","last_fullsized":"2024-09-21T10:03:53.830275+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:32.280697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.840992+0000","last_change":"2024-09-21T10:03:53.840992+0000","last_active":"2024-09-21T10:03:53.840992+0000","last_peered":"2024-09-21T10:03:53.840992+0000","last_clean":"2024-09-21T10:03:53.840992+0000","last_became_active":"2024-09-21T10:03:53.840551+0000","last_became_peered":"2024-09-21T10:03:53.840551+0000","last_unstale":"2024-09-21T10:03:53.840992+0000","last_undegraded":"2024-09-21T10:03:53.840992+0000","last_fullsized":"2024-09-21T10:03:53.840992+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:04.442126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.836717+0000","last_change":"2024-09-21T10:02:46.659513+0000","last_active":"2024-09-21T10:03:52.836717+0000","last_peered":"2024-09-21T10:03:52.836717+0000","last_clean":"2024-09-21T10:03:52.836717+0000","last_became_active":"2024-09-21T10:02:46.659130+0000","last_became_peered":"2024-09-21T10:02:46.659130+0000","last_unstale":"2024-09-21T10:03:52.836717+0000","last_undegraded":"2024-09-21T10:03:52.836717+0000","last_fullsized":"2024-09-21T10:03:52.836717+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:24.792175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.835006+0000","last_change":"2024-09-21T10:03:53.835006+0000","last_active":"2024-09-21T10:03:53.835006+0000","last_peered":"2024-09-21T10:03:53.835006+0000","last_clean":"2024-09-21T10:03:53.835006+0000","last_became_active":"2024-09-21T10:03:53.834608+0000","last_became_peered":"2024-09-21T10:03:53.834608+0000","last_unstale":"2024-09-21T10:03:53.835006+0000","last_undegraded":"2024-09-21T10:03:53.835006+0000","last_fullsized":"2024-09-21T10:03:53.835006+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:56.108606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":59,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-09-21T10:03:52.837866+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:03:52.837866+0000","last_peered":"2024-09-21T10:03:52.837866+0000","last_clean":"2024-09-21T10:03:52.837866+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:03:52.837866+0000","last_undegraded":"2024-09-21T10:03:52.837866+0000","last_fullsized":"2024-09-21T10:03:52.837866+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.844128+0000","last_change":"2024-09-21T10:03:53.844043+0000","last_active":"2024-09-21T10:03:53.844128+0000","last_peered":"2024-09-21T10:03:53.844128+0000","last_clean":"2024-09-21T10:03:53.844128+0000","last_became_active":"2024-09-21T10:03:53.842891+0000","last_became_peered":"2024-09-21T10:03:53.842891+0000","last_unstale":"2024-09-21T10:03:53.844128+0000","last_undegraded":"2024-09-21T10:03:53.844128+0000","last_fullsized":"2024-09-21T10:03:53.844128+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:39.891977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.842964+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:03:53.842964+0000","last_peered":"2024-09-21T10:03:53.842964+0000","last_clean":"2024-09-21T10:03:53.842964+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:03:53.842964+0000","last_undegraded":"2024-09-21T10:03:53.842964+0000","last_fullsized":"2024-09-21T10:03:53.842964+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.831425+0000","last_change":"2024-09-21T10:02:46.659818+0000","last_active":"2024-09-21T10:03:53.831425+0000","last_peered":"2024-09-21T10:03:53.831425+0000","last_clean":"2024-09-21T10:03:53.831425+0000","last_became_active":"2024-09-21T10:02:46.659457+0000","last_became_peered":"2024-09-21T10:02:46.659457+0000","last_unstale":"2024-09-21T10:03:53.831425+0000","last_undegraded":"2024-09-21T10:03:53.831425+0000","last_fullsized":"2024-09-21T10:03:53.831425+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:21.962330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-09-21T10:03:53.830391+0000","last_change":"2024-09-21T10:03:19.769549+0000","last_active":"2024-09-21T10:03:53.830391+0000","last_peered":"2024-09-21T10:03:53.830391+0000","last_clean":"2024-09-21T10:03:53.830391+0000","last_became_active":"2024-09-21T10:03:19.768854+0000","last_became_peered":"2024-09-21T10:03:19.768854+0000","last_unstale":"2024-09-21T10:03:53.830391+0000","last_undegraded":"2024-09-21T10:03:53.830391+0000","last_fullsized":"2024-09-21T10:03:53.830391+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:29.415039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234183,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37008,"kb_used_data":5884,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708144,"statfs":{"total":95995035648,"available":95957139456,"internally_reserved":0,"allocated":6025216,"data_stored":5417204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495821,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32908,"kb_used_data":1784,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712244,"statfs":{"total":95995035648,"available":95961337856,"internally_reserved":0,"allocated":1826816,"data_stored":1223104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757458,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28216,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716936,"statfs":{"total":95995035648,"available":95966142464,"internally_reserved":0,"allocated":1224704,"data_stored":628562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019094,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32332,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712820,"statfs":{"total":95995035648,"available":95961927680,"internally_reserved":0,"allocated":1236992,"data_stored":632793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435278,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32888,"kb_used_data":1188,"kb_used_omap":5,"kb_used_meta":31674,"kb_avail":93712264,"statfs":{"total":95995035648,"available":95961358336,"internally_reserved":0,"allocated":1216512,"data_stored":627993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":32434216},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598805,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1188,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1216512,"data_stored":628211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":28172749},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762332,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36976,"kb_used_data":5280,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708176,"statfs":{"total":95995035648,"available":95957172224,"internally_reserved":0,"allocated":5406720,"data_stored":4821953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":32432381},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925859,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33404,"kb_used_data":5868,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711748,"statfs":{"total":95995035648,"available":95960829952,"internally_reserved":0,"allocated":6008832,"data_stored":5413179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":28172478},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:04:58.843 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T10:04:58.843 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump 2024-09-21T10:04:59.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:58 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:59.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:58 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:04:59.733 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:59.844 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:04:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:04:59] "GET /metrics HTTP/1.1" 200 33920 "" "Prometheus/2.51.0" 2024-09-21T10:04:59.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:59 smithi032 ceph-mon[101299]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:59.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:04:59 smithi032 ceph-mon[101299]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:04:59.844 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:59 smithi032 ceph-mon[105748]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:59.844 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:04:59 smithi032 ceph-mon[105748]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:59 smithi045 ceph-mon[95976]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:04:59 smithi045 ceph-mon[95976]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:01.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:01 smithi032 ceph-mon[101299]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:01.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:01 smithi032 ceph-mon[105748]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:02.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:01 smithi045 ceph-mon[95976]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:02.496 INFO:teuthology.orchestra.run.smithi032.stdout:epoch 110 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:created 2024-09-21T09:43:37.317307+0000 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:modified 2024-09-21T10:03:53.826703+0000 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:crush_version 22 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:full_ratio 0.95 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:backfillfull_ratio 0.9 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:nearfull_ratio 0.85 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:require_min_compat_client luminous 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:min_compat_client luminous 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:require_osd_release reef 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:stretch_mode_enabled false 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-09-21T10:05:02.497 INFO:teuthology.orchestra.run.smithi032.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 85 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 70 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:max_osd 8 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103] [v2:172.21.15.32:6804/4212241103,v1:172.21.15.32:6805/4212241103] exists,up 0f6f5dda-c134-4283-a91d-cf501ef19062 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778] [v2:172.21.15.32:6812/1357153778,v1:172.21.15.32:6813/1357153778] exists,up bdaa43fd-b969-4d06-b6eb-97b415c4e118 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490] [v2:172.21.15.32:6820/3097218490,v1:172.21.15.32:6821/3097218490] exists,up 1f12d9f2-90b0-44f6-8432-0b398194b3b3 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087] [v2:172.21.15.32:6828/438066087,v1:172.21.15.32:6829/438066087] exists,up 6a568fd5-5fd9-46f8-b3af-9fa196ed617b 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252] [v2:172.21.15.45:6802/413300252,v1:172.21.15.45:6803/413300252] exists,up 2cf149e0-4c2f-46eb-9ca2-a51907aea3b0 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 up in weight 1 up_from 38 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729] [v2:172.21.15.45:6810/3168849729,v1:172.21.15.45:6811/3168849729] exists,up a6c78855-6756-4b0b-9a9c-8d102079540b 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 up in weight 1 up_from 46 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679] [v2:172.21.15.45:6818/2158139679,v1:172.21.15.45:6819/2158139679] exists,up d476169d-dffc-4f1e-b60b-4f836f53285e 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 up in weight 1 up_from 54 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350] [v2:172.21.15.45:6826/3336053350,v1:172.21.15.45:6827/3336053350] exists,up 021a4fd0-f770-4799-9683-b0aa7cd593bc 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 4.10 [1,2] 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 4.12 [1,5] 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:pg_upmap_items 4.1f [1,2] 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/3511996666 expires 2024-09-22T09:56:17.980985+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/1200491506 expires 2024-09-22T09:53:08.870463+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/1465894634 expires 2024-09-22T09:53:08.870463+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6801/2033554920 expires 2024-09-22T09:49:07.125921+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6801/4033152109 expires 2024-09-22T09:44:29.817113+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.45:0/2114035806 expires 2024-09-22T09:53:38.533217+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6801/2819046815 expires 2024-09-22T09:43:58.900171+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.45:0/102537314 expires 2024-09-22T09:53:38.533217+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/458122323 expires 2024-09-22T09:49:07.125921+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.45:6835/1148645562 expires 2024-09-22T09:53:38.533217+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.45:0/599831880 expires 2024-09-22T09:53:38.533217+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/1807347540 expires 2024-09-22T09:56:17.980985+0000 2024-09-21T10:05:02.499 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/230160407 expires 2024-09-22T09:43:58.900171+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6800/4033152109 expires 2024-09-22T09:44:29.817113+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/2785183325 expires 2024-09-22T09:49:07.125921+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6801/431176591 expires 2024-09-22T09:53:08.870463+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/2408427509 expires 2024-09-22T09:56:17.980985+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/1178553739 expires 2024-09-22T09:43:58.900171+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6800/431176591 expires 2024-09-22T09:53:08.870463+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/2103196055 expires 2024-09-22T09:53:08.870463+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/2042031504 expires 2024-09-22T09:44:29.817113+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/1259145858 expires 2024-09-22T09:49:07.125921+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/533058733 expires 2024-09-22T09:44:29.817113+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/3332221400 expires 2024-09-22T09:43:58.900171+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.45:0/2100270146 expires 2024-09-22T09:53:38.533217+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/1102447792 expires 2024-09-22T09:56:17.980985+0000 2024-09-21T10:05:02.500 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.45:0/431361432 expires 2024-09-22T09:53:38.533217+0000 2024-09-21T10:05:02.501 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6800/2033554920 expires 2024-09-22T09:49:07.125921+0000 2024-09-21T10:05:02.501 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6801/2805220519 expires 2024-09-22T09:56:17.980985+0000 2024-09-21T10:05:02.501 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/1858508442 expires 2024-09-22T09:53:08.870463+0000 2024-09-21T10:05:02.501 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/2895073807 expires 2024-09-22T09:44:29.817113+0000 2024-09-21T10:05:02.501 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6800/2819046815 expires 2024-09-22T09:43:58.900171+0000 2024-09-21T10:05:02.501 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:6800/2805220519 expires 2024-09-22T09:56:17.980985+0000 2024-09-21T10:05:02.501 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.45:0/338006180 expires 2024-09-22T09:53:38.533217+0000 2024-09-21T10:05:02.501 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.32:0/2388392475 expires 2024-09-22T09:49:07.125921+0000 2024-09-21T10:05:02.501 INFO:teuthology.orchestra.run.smithi032.stdout:blocklist 172.21.15.45:6834/1148645562 expires 2024-09-22T09:53:38.533217+0000 2024-09-21T10:05:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:03 smithi045 ceph-mon[95976]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:05:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:05:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:03 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4078732297' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[105748]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4078732297' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[101299]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:03.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:05:03.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:05:03.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:03.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:03 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4078732297' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-21T10:05:03.356 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.32:6802/4212241103,v1:172.21.15.32:6803/4212241103] [v2:172.21.15.32:6804/4212241103,v1:172.21.15.32:6805/4212241103] exists,up 0f6f5dda-c134-4283-a91d-cf501ef19062', 'osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.32:6810/1357153778,v1:172.21.15.32:6811/1357153778] [v2:172.21.15.32:6812/1357153778,v1:172.21.15.32:6813/1357153778] exists,up bdaa43fd-b969-4d06-b6eb-97b415c4e118', 'osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.32:6818/3097218490,v1:172.21.15.32:6819/3097218490] [v2:172.21.15.32:6820/3097218490,v1:172.21.15.32:6821/3097218490] exists,up 1f12d9f2-90b0-44f6-8432-0b398194b3b3', 'osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.32:6826/438066087,v1:172.21.15.32:6827/438066087] [v2:172.21.15.32:6828/438066087,v1:172.21.15.32:6829/438066087] exists,up 6a568fd5-5fd9-46f8-b3af-9fa196ed617b', 'osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6800/413300252,v1:172.21.15.45:6801/413300252] [v2:172.21.15.45:6802/413300252,v1:172.21.15.45:6803/413300252] exists,up 2cf149e0-4c2f-46eb-9ca2-a51907aea3b0', 'osd.5 up in weight 1 up_from 38 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6808/3168849729,v1:172.21.15.45:6809/3168849729] [v2:172.21.15.45:6810/3168849729,v1:172.21.15.45:6811/3168849729] exists,up a6c78855-6756-4b0b-9a9c-8d102079540b', 'osd.6 up in weight 1 up_from 46 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6816/2158139679,v1:172.21.15.45:6817/2158139679] [v2:172.21.15.45:6818/2158139679,v1:172.21.15.45:6819/2158139679] exists,up d476169d-dffc-4f1e-b60b-4f836f53285e', 'osd.7 up in weight 1 up_from 54 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6824/3336053350,v1:172.21.15.45:6825/3336053350] [v2:172.21.15.45:6826/3336053350,v1:172.21.15.45:6827/3336053350] exists,up 021a4fd0-f770-4799-9683-b0aa7cd593bc'] 2024-09-21T10:05:03.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon mon.a config show 2024-09-21T10:05:03.567 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:04.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout: "name": "mon.a", 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster": "ceph", 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout: "admin_socket_mode": "", 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout: "auth_client_required": "cephx, none", 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout: "auth_cluster_required": "cephx", 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout: "auth_debug": "true", 2024-09-21T10:05:05.217 INFO:teuthology.orchestra.run.smithi032.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "auth_service_required": "cephx", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "auth_service_ticket_ttl": "240.000000", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "auth_supported": "", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_aio": "true", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_aio_max_queue_depth": "1024", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_aio_poll_ms": "250", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_aio_reap_max": "16", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_async_discard_threads": "0", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_block_size": "4096", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_debug_aio": "false", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-09-21T10:05:05.218 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_debug_inflight_ios": "false", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_enable_discard": "false", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_flock_retry": "3", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_flock_retry_interval": "0.100000", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_inject_crash": "0", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_inject_crash_flush_delay": "2", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_ioring": "false", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_ioring_hipri": "false", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_ioring_sqthread_poll": "false", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_read_buffer_alignment": "4096", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-09-21T10:05:05.219 INFO:teuthology.orchestra.run.smithi032.stdout: "bdev_type": "", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_alloc_size": "1048576", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_allocator": "hybrid", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_buffered_io": "true", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_check_for_zeros": "false", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_check_volume_selector_often": "false", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_compact_log_sync": "false", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_log_compact_min_size": "16777216", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_log_replay_check_allocations": "true", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_max_log_runway": "4194304", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_max_prefetch": "1048576", 2024-09-21T10:05:05.220 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_min_flush_size": "524288", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_min_log_runway": "1048576", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_replay_recovery": "false", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_shared_alloc_size": "65536", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluefs_sync_write": "false", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_allocation_from_file": "true", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_allocator": "hybrid", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_async_db_compaction": "true", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-09-21T10:05:05.221 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_bdev_label_multi": "true", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_bdev_label_require_all": "true", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_blobid_prealloc": "10240", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_create": "true", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_db_create": "false", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_db_path": "", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_db_size": "0", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_path": "", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_preallocate_file": "false", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_size": "107374182400", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_wal_create": "false", 2024-09-21T10:05:05.222 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_wal_path": "", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_block_wal_size": "100663296", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_bluefs": "true", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_bluefs_env_mirror": "false", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_autotune": "true", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-09-21T10:05:05.223 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_size": "0", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cache_type": "2q", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_clone_cow": "true", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_compression_algorithm": "snappy", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_compression_max_blob_size": "0", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-09-21T10:05:05.224 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_compression_min_blob_size": "0", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_compression_mode": "none", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_csum_type": "crc32c", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_enforce_settings": "default", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_freelist": "false", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_fsck_abort": "false", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_inject_read_err": "false", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_legacy_omap": "false", 2024-09-21T10:05:05.225 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_small_allocations": "0", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_default_buffered_read": "true", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_default_buffered_write": "false", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_deferred_batch_ops": "0", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_elastic_shared_blobs": "true", 2024-09-21T10:05:05.226 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fail_eio": "false", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_on_mkfs": "true", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_on_mount": "false", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_on_umount": "false", 2024-09-21T10:05:05.227 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_ignore_data_csum": "false", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_kvbackend": "rocksdb", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_log_op_age": "5.000000", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_max_alloc_size": "0", 2024-09-21T10:05:05.228 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_max_blob_size": "0", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_max_defer_interval": "3.000000", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_max_deferred_txc": "32", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_min_alloc_size": "0", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_nid_prealloc": "1024", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_prefer_deferred_size": "0", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_qfsck_on_mount": "true", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_retry_disk_reads": "3", 2024-09-21T10:05:05.229 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_rocksdb_cf": "true", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_rocksdb_options_annex": "", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_spdk_coremask": "0x1", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_spdk_io_sleep": "5", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_spdk_max_io_completion": "0", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_spdk_mem": "512", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_sync_submit_transaction": "false", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_throttle_bytes": "67108864", 2024-09-21T10:05:05.230 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_throttle_cost_per_io": "0", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_tracing": "false", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_volume_selection_reserved": "0", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-09-21T10:05:05.231 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_write_v2": "false", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_write_v2_random": "false", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "bluestore_zero_block_detection": "false", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_action_update_interval": "2", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_mount_timeout": "10", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-09-21T10:05:05.232 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephx_cluster_require_signatures": "false", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephx_cluster_require_version": "2", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephx_require_signatures": "false", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephx_require_version": "2", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephx_service_require_signatures": "false", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephx_service_require_version": "2", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "cephx_sign_messages": "true", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "chdir": "", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "client_acl_type": "", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "client_asio_thread_count": "2", 2024-09-21T10:05:05.233 INFO:teuthology.orchestra.run.smithi032.stdout: "client_cache_mid": "0.750000", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_cache_size": "16384", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_caps_release_delay": "5", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_check_pool_perm": "true", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_collect_and_send_global_metrics": "false", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_debug_force_sync_read": "false", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_debug_getattr_caps": "false", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_debug_inject_features": "", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_debug_inject_tick_delay": "0", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-09-21T10:05:05.234 INFO:teuthology.orchestra.run.smithi032.stdout: "client_die_on_failed_remount": "false", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_dirsize_rbytes": "true", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_force_lazyio": "false", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_fs": "", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_inject_fixed_oldest_tid": "false", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_inject_release_failure": "false", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_max_inline_size": "4096", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_max_retries_on_remount_failure": "5", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_mds_namespace": "", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_metadata": "", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_mount_gid": "-1", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_mount_timeout": "300", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_mount_uid": "-1", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_mountpoint": "/", 2024-09-21T10:05:05.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_notify_timeout": "10", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_oc": "true", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_oc_max_dirty": "104857600", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_oc_max_dirty_age": "5.000000", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_oc_max_objects": "1000", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_oc_size": "209715200", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_oc_target_dirty": "8388608", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_permissions": "true", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_quota": "true", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_quota_df": "true", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_readahead_max_bytes": "0", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_readahead_max_periods": "4", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_readahead_min": "131072", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_reconnect_stale": "false", 2024-09-21T10:05:05.236 INFO:teuthology.orchestra.run.smithi032.stdout: "client_shutdown_timeout": "30", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "client_snapdir": ".snap", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "client_tick_interval": "1", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "client_trace": "", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "client_try_dentry_invalidate": "false", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "client_use_faked_inos": "false", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "client_use_random_mds": "false", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "clog_to_graylog": "false", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "clog_to_graylog_port": "12201", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "clog_to_monitors": "default=true", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "clog_to_syslog": "false", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "clog_to_syslog_level": "info", 2024-09-21T10:05:05.237 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_addr": "-", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_network": "", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster_network_interface": "", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "compressor_zlib_isal": "false", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "compressor_zlib_level": "5", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "compressor_zlib_winsize": "-15", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "compressor_zstd_level": "1", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "crimson_alien_op_num_threads": "6", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "crimson_alien_thread_cpu_cores": "", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "crimson_osd_obc_lru_size": "512", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "crimson_osd_stat_interval": "0", 2024-09-21T10:05:05.238 INFO:teuthology.orchestra.run.smithi032.stdout: "crimson_seastar_cpu_cores": "", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "crimson_seastar_num_threads": "0", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_location": "", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_location_hook": "", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "crush_location_hook_timeout": "10", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "daemonize": "false", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "daos_pool": "tank", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "dbstore_db_name_prefix": "dbstore", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_alienstore": "0/5", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_allow_any_pool_priority": "false", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_asok": "1/5", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_asok_assert_abort": "false", 2024-09-21T10:05:05.239 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_asserts_on_shutdown": "true", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_auth": "1/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_bdev": "1/3", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_bluefs": "1/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_bluestore": "1/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_buffer": "0/1", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_ceph_dedup": "0/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_ceph_exporter": "1/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_cephfs_mirror": "0/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_cephsqlite": "0/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_client": "0/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_compressor": "1/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_context": "0/1", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_crimson_interrupt": "0/5", 2024-09-21T10:05:05.240 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_crush": "1/1", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_crypto": "1/5", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_cyanstore": "0/5", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_deliberately_leak_memory": "false", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_disable_randomized_ping": "false", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_dpdk": "1/5", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_eventtrace": "1/5", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_filer": "0/1", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_filestore": "1/3", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_finisher": "1/1", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_fuse": "1/5", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_heartbeat_testing_span": "0", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_heartbeatmap": "1/5", 2024-09-21T10:05:05.241 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_immutable_obj_cache": "0/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_javaclient": "1/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_journal": "1/3", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_journaler": "0/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_kstore": "1/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_lockdep": "0/1", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mclock": "1/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mds": "1/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mds_balancer": "1/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mds_locker": "1/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mds_log": "1/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mds_log_expire": "1/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mds_migrator": "1/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mds_quiesce": "3/5", 2024-09-21T10:05:05.242 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_memstore": "1/5", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mgr": "2/5", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mgrc": "1/5", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_mon": "20/20", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_monc": "0/10", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_ms": "1/1", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_none": "0/5", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_objclass": "0/5", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_objectcacher": "0/5", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_objecter": "0/1", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_optracker": "0/5", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_osd": "1/5", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_paxos": "20/20", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_perfcounter": "1/5", 2024-09-21T10:05:05.243 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_prioritycache": "1/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rados": "0/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rbd": "0/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rbd_mirror": "0/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rbd_pwl": "0/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rbd_replay": "0/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_refs": "0/0", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_reserver": "1/1", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rgw": "1/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rgw_access": "1/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rgw_datacache": "1/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rgw_dbstore": "1/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rgw_flight": "1/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rgw_lifecycle": "1/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rgw_notification": "1/5", 2024-09-21T10:05:05.244 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rgw_sync": "1/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_rocksdb": "4/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_backref": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_cache": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_cleaner": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_device": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_epm": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_journal": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_lba": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_odata": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_omap": "0/5", 2024-09-21T10:05:05.245 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_onode": "0/5", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_t": "0/5", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_seastore_tm": "0/5", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_striper": "0/1", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_test": "0/5", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_throttle": "1/1", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_timer": "0/1", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_tp": "0/5", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "debug_trace": "1/5", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "device_failure_prediction_mode": "none", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "err_to_graylog": "false", 2024-09-21T10:05:05.246 INFO:teuthology.orchestra.run.smithi032.stdout: "err_to_journald": "false", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "err_to_stderr": "true", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "err_to_syslog": "false", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "event_tracing": "false", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "exporter_addr": "0.0.0.0", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "exporter_cert_file": "", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "exporter_http_port": "9926", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "exporter_key_file": "", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "exporter_prio_limit": "5", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "exporter_sort_metrics": "true", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "exporter_stats_period": "5", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "fake_statfs_for_testing": "0", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "fatal_signal_handlers": "true", 2024-09-21T10:05:05.247 INFO:teuthology.orchestra.run.smithi032.stdout: "filer_max_purge_ops": "10", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filer_max_truncate_ops": "128", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_apply_finisher_threads": "1", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_blackhole": "false", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_btrfs_clone_range": "true", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_btrfs_snap": "true", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_caller_concurrency": "10", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_collect_device_partition_information": "true", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_commit_timeout": "600.000000", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_debug_inject_read_err": "false", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_debug_omap_check": "false", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_debug_verify_split": "false", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_dump_file": "", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-09-21T10:05:05.248 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_fadvise": "true", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_fail_eio": "true", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_fd_cache_shards": "16", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_fd_cache_size": "128", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_fiemap": "false", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_fiemap_threshold": "4096", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_index_retry_probability": "0.000000", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_inject_stall": "0", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_journal_parallel": "false", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_journal_trailing": "false", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_journal_writeahead": "false", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_kill_at": "0", 2024-09-21T10:05:05.249 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_inline_xattr_size": "0", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_inline_xattrs": "0", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_inline_xattrs_other": "2", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_sync_interval": "5.000000", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_xattr_value_size": "0", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-09-21T10:05:05.250 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_merge_threshold": "-10", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_min_sync_interval": "0.010000", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_odsync_write": "false", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_omap_backend": "rocksdb", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_omap_backend_path": "", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_omap_header_cache_size": "1024", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_ondisk_finisher_threads": "1", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_op_thread_timeout": "60", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_op_threads": "2", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_punch_hole": "false", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_max_bytes": "104857600", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_queue_max_ops": "50", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_seek_data_hole": "false", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_sloppy_crc": "false", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_splice": "false", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_split_multiple": "2", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_split_rand_factor": "20", 2024-09-21T10:05:05.252 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_update_to": "1000", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_enable": "true", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-09-21T10:05:05.253 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_xfs_extsize": "false", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "filestore_zfs_snap": "false", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fio_dir": "/tmp/fio", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fsid": "e0bde4d4-77fd-11ef-baf6-efdc52797490", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_allow_other": "true", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_atomic_o_trunc": "true", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_big_writes": "true", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_debug": "false", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_default_permissions": "false", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_disable_pagecache": "false", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_max_write": "0", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_multithreaded": "true", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_require_active_mds": "true", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_set_user_groups": "true", 2024-09-21T10:05:05.254 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_splice_move": "true", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_splice_read": "true", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_splice_write": "true", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_syncfs_on_mksnap": "true", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "fuse_use_invalidate_cb": "true", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "gss_target_name": "ceph", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "heartbeat_file": "", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "heartbeat_inject_failure": "0", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "heartbeat_interval": "5", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "host": "smithi032", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_max_size": "1073741824", 2024-09-21T10:05:05.255 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "immutable_object_cache_watermark": "0.900000", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "inject_early_sigterm": "false", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "jaeger_agent_port": "6799", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "jaeger_tracing_enable": "false", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_aio": "true", 2024-09-21T10:05:05.256 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_align_min_size": "65536", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_block_align": "true", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_block_size": "4096", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_dio": "true", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_discard": "false", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_force_aio": "false", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_ignore_corruption": "false", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_max_write_bytes": "10485760", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_max_write_entries": "100", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_replay_from": "0", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_throttle_high_multiple": "0.000000", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_throttle_max_multiple": "0.000000", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_write_header_frequency": "0", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journal_zero_on_create": "false", 2024-09-21T10:05:05.257 INFO:teuthology.orchestra.run.smithi032.stdout: "journaler_prefetch_periods": "10", 2024-09-21T10:05:05.258 INFO:teuthology.orchestra.run.smithi032.stdout: "journaler_prezero_periods": "5", 2024-09-21T10:05:05.258 INFO:teuthology.orchestra.run.smithi032.stdout: "journaler_write_head_interval": "15", 2024-09-21T10:05:05.258 INFO:teuthology.orchestra.run.smithi032.stdout: "key": "", 2024-09-21T10:05:05.258 INFO:teuthology.orchestra.run.smithi032.stdout: "keyfile": "", 2024-09-21T10:05:05.258 INFO:teuthology.orchestra.run.smithi032.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-09-21T10:05:05.258 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_backend": "rocksdb", 2024-09-21T10:05:05.258 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_default_stripe_size": "65536", 2024-09-21T10:05:05.258 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_fsck_on_mount": "false", 2024-09-21T10:05:05.259 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_fsck_on_mount_deep": "true", 2024-09-21T10:05:05.259 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_max_bytes": "67108864", 2024-09-21T10:05:05.259 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_max_ops": "512", 2024-09-21T10:05:05.259 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_nid_prealloc": "1024", 2024-09-21T10:05:05.259 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_onode_map_size": "1024", 2024-09-21T10:05:05.259 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-09-21T10:05:05.259 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_sync_submit_transaction": "false", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "kstore_sync_transaction": "false", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "librados_thread_count": "2", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "lockdep": "false", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "lockdep_force_backtrace": "false", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_coarse_timestamps": "true", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_flush_on_exit": "false", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_graylog_host": "127.0.0.1", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_graylog_port": "12201", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_max_new": "1000", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_max_recent": "10000", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_stderr_prefix": "", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_stop_at_utilization": "0.970000", 2024-09-21T10:05:05.260 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_file": "true", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_graylog": "false", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_journald": "false", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_stderr": "false", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "log_to_syslog": "false", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "max_rotating_auth_attempts": "10", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_action_on_write_error": "1", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_alternate_name_max": "8192", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_asio_thread_count": "2", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_export_pin": "true", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_fragment_dirs": "true", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_fragment_interval": "5", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_fragment_size_max": "100000", 2024-09-21T10:05:05.261 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_idle_threshold": "0.000000", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_interval": "10", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_max": "-1", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_max_until": "-1", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_merge_size": "50", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_midchunk": "0.300000", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_min_rebalance": "0.100000", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_min_start": "0.200000", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_minchunk": "0.001000", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_mode": "0", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_need_max": "1.200000", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_need_min": "0.800000", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_overload_epochs": "2", 2024-09-21T10:05:05.262 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_sample_interval": "3.000000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_split_bits": "3", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_split_rd": "25000.000000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_split_size": "10000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_split_wr": "10000.000000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_target_decay": "10.000000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_beacon_grace": "15.000000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_beacon_interval": "4.000000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_beacon_mon_down_grace": "60", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_memory_limit": "4294967296", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_mid": "0.700000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-09-21T10:05:05.263 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_quiesce_delay": "0", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_quiesce_sleep": "200", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_quiesce_splitauth": "true", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_quiesce_threshold": "524288", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_release_free_interval": "10", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_reservation": "0.050000", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_trim_interval": "1", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cache_trim_threshold": "262144", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_client_delegate_inos_pct": "50", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_client_prealloc_inos": "1000", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_connect_bootstrapping": "false", 2024-09-21T10:05:05.264 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_damage_table_max_entries": "10000", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_debug_auth_pins": "false", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_debug_frag": "false", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_debug_scatterstat": "false", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_debug_subtrees": "false", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_decay_halflife": "5.000000", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_default_dir_hash": "2", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_defer_session_stale": "true", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_deny_all_reconnect": "false", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_dir_keys_per_op": "16384", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_dir_max_commit_size": "10", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_dir_max_entries": "0", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_dir_prefetch": "true", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_dirstat_min_interval": "1.000000", 2024-09-21T10:05:05.265 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_dump_cache_after_rejoin": "false", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_dump_cache_on_map": "false", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_dump_cache_threshold_file": "0", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_early_reply": "true", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_enable_op_tracker": "true", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_enforce_unique_name": "true", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_export_ephemeral_distributed": "true", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_export_ephemeral_random": "true", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_forward_all_requests_to_auth": "false", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-09-21T10:05:05.266 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_health_cache_threshold": "1.500000", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_health_summarize_threshold": "10", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_heartbeat_grace": "15.000000", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_heartbeat_reset_grace": "1000", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_inject_health_dummy": "false", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_inject_migrator_session_race": "false", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_join_fs": "", 2024-09-21T10:05:05.267 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_journal_format": "1", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_create_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_dirfrag_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_export_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_import_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_journal_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_journal_expire_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_journal_replay_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_link_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_mdstable_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_openc_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_rename_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_kill_shutdown_at": "0", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_event_large_threshold": "524288", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_events_per_segment": "1024", 2024-09-21T10:05:05.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_major_segment_event_ratio": "12", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_max_events": "-1", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_max_segments": "128", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_pause": "false", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_segment_size": "0", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_skip_corrupt_events": "false", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_skip_unbounded_events": "false", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_trim_threshold": "128", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_log_warn_factor": "2.000000", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_caps_per_client": "1048576", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_completed_flushes": "100000", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_completed_requests": "100000", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_export_size": "20971520", 2024-09-21T10:05:05.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_file_recover": "32", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_purge_files": "64", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_purge_ops": "8192", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_max_snaps_per_dir": "100", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_metrics_update_interval": "2", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_min_caps_per_client": "100", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_min_caps_working_set": "10000", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_numa_node": "-1", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_op_complaint_time": "30.000000", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_op_history_duration": "600", 2024-09-21T10:05:05.270 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_op_history_size": "20", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_op_history_slow_op_size": "20", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_op_history_slow_op_threshold": "10", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_op_log_threshold": "5", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_ping_grace": "15", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_ping_interval": "5", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_recall_max_caps": "30000", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_recall_max_decay_threshold": "131072", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_recall_warning_threshold": "262144", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_reconnect_timeout": "45.000000", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_replay_interval": "1.000000", 2024-09-21T10:05:05.271 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_root_ino_gid": "0", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_root_ino_uid": "0", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_scrub_stats_review_period": "1", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_session_blocklist_on_evict": "true", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_session_blocklist_on_timeout": "true", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-09-21T10:05:05.272 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_session_metadata_threshold": "16777216", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_shutdown_check": "0", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_skip_ino": "0", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_sleep_rank_change": "0.000000", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_snap_max_uid": "4294967294", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_snap_min_uid": "0", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_snap_rstat": "false", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_standby_replay_damaged": "false", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_symlink_recovery": "true", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_task_status_update_interval": "2.000000", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_thrash_exports": "0", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_thrash_fragments": "0", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_tick_interval": "5.000000", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_valgrind_exit": "false", 2024-09-21T10:05:05.273 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_verify_backtrace": "1", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_verify_scatter": "false", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_wipe_ino_prealloc": "false", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mds_wipe_sessions": "false", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mempool_debug": "false", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "memstore_debug_omit_block_device_write": "false", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "memstore_device_bytes": "1073741824", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "memstore_page_set": "false", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "memstore_page_size": "65536", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_client_bytes": "134217728", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_client_messages": "512", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_connect_retry_interval": "1.000000", 2024-09-21T10:05:05.274 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_disabled_modules": "", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_enable_op_tracker": "true", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_max_pg_creating": "1024", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_max_pg_num_change": "128", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_mds_bytes": "134217728", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_mds_messages": "128", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_mon_bytes": "134217728", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_mon_messages": "128", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_num_op_tracker_shard": "32", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_op_complaint_time": "30.000000", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_op_history_duration": "600", 2024-09-21T10:05:05.275 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_op_history_size": "20", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_op_history_slow_op_size": "20", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_op_log_threshold": "5", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_osd_bytes": "536870912", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_osd_messages": "8192", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_pool": "true", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_service_beacon_grace": "60.000000", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_standby_modules": "true", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_stats_period": "5", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_stats_threshold": "5", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_tick_period": "2", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_accept_timeout_factor": "2.000000", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_allow_pool_delete": "true", 2024-09-21T10:05:05.276 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_allow_pool_size_one": "false", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_auth_validate_all_caps": "true", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_bytes": "104857600", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_directed_command_retry": "2", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_hunt_interval": "3.000000", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_hunt_parallel": "3", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_log_interval": "1.000000", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_ping_interval": "10.000000", 2024-09-21T10:05:05.277 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_ping_timeout": "30.000000", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_client_target_rank": "-1", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_clock_drift_allowed": "1.000000", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_level": "debug", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_to_file": "true", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_to_graylog": "false", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_to_journald": "true", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_to_stderr": "false", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-09-21T10:05:05.278 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_compact_on_bootstrap": "false", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_compact_on_start": "false", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_compact_on_trim": "true", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_con_tracker_persist_interval": "10", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_con_tracker_score_halflife": "43200", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_config_key_max_entry_size": "65536", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_cpu_threads": "4", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_crush_min_required_version": "hammer", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_daemon_bytes": "419430400", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_data_avail_crit": "5", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_data_avail_warn": "5", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_data_size_warn": "16106127360", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_block_osdmap_trim": "false", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_dump_json": "false", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_dump_transactions": "false", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_extra_checks": "false", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_no_require_reef": "false", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_no_require_squid": "false", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_delta_reset_interval": "10.000000", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_dns_srv_name": "ceph-mon", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_down_mkfs_grace": "60", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_down_uptime_grace": "60", 2024-09-21T10:05:05.280 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_election_default_strategy": "3", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_election_timeout": "5.000000", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_elector_ping_divisor": "2", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_elector_ping_timeout": "2.000000", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_enable_op_tracker": "true", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_fake_pool_delete": "false", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_force_quorum_join": "false", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_fsmap_prune_threshold": "300", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_globalid_prealloc": "10000", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_health_detail_to_clog": "true", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_health_log_update_period": "5", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_health_max_detail": "50", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_health_to_clog": "true", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_health_to_clog_interval": "600", 2024-09-21T10:05:05.281 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_host": "[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] [v2:172.21.15.32:3301/0,v1:172.21.15.32:6790/0]", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_host_override": "", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_initial_members": "", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_keyvaluedb": "rocksdb", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_lease": "5.000000", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_log_full_interval": "50", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_log_max": "10000", 2024-09-21T10:05:05.282 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_log_max_summary": "50", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_max_log_entries_per_event": "4096", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_max_log_epochs": "500", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_max_mdsmap_epochs": "500", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_max_mgrmap_epochs": "500", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_max_nvmeof_epochs": "500", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_max_osd": "10000", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_max_pg_per_osd": "10000", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_max_pool_pg_num": "65536", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_mds_force_trim_to": "0", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_mds_skip_sanity": "true", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_memory_autotune": "true", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_memory_target": "2147483648", 2024-09-21T10:05:05.283 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_mgr_beacon_grace": "30", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_mgr_digest_period": "5", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_mgr_inactive_grace": "60", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_mgr_mkfs_grace": "240", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_min_osdmap_epochs": "500", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_op_complaint_time": "30", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_op_history_duration": "600", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_op_history_size": "20", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_op_history_slow_op_size": "20", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_op_history_slow_op_threshold": "10", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_op_log_threshold": "5", 2024-09-21T10:05:05.284 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_auto_mark_in": "false", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_auto_mark_new_in": "true", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_cache_size": "500", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_cache_size_min": "134217728", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_crush_smoke_test": "true", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_destroyed_out_interval": "600", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_down_out_interval": "600", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-09-21T10:05:05.285 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_force_trim_to": "0", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_full_ratio": "0.950000", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_laggy_halflife": "3600", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_laggy_max_interval": "300", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_laggy_weight": "0.300000", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_max_initial_pgs": "1024", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_min_down_reporters": "2", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_prime_pg_temp": "true", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-09-21T10:05:05.286 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-09-21T10:05:05.287 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_report_timeout": "900", 2024-09-21T10:05:05.287 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:05 smithi032 ceph-mon[101299]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:05.287 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:05 smithi032 ceph-mon[105748]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:05.288 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-09-21T10:05:05.288 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-09-21T10:05:05.288 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_warn_num_repaired": "10", 2024-09-21T10:05:05.288 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osd_warn_op_age": "32.000000", 2024-09-21T10:05:05.288 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-09-21T10:05:05.288 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osdmap_full_prune_interval": "10", 2024-09-21T10:05:05.288 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osdmap_full_prune_min": "10000", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_pg_stuck_threshold": "60", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_pg_warn_min_objects": "10000", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_pg_warn_min_per_osd": "0", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_pool_quota_crit_threshold": "0", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_pool_quota_warn_threshold": "0", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_probe_timeout": "2.000000", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_reweight_max_change": "0.050000", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_reweight_max_osds": "4", 2024-09-21T10:05:05.289 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_scrub_interval": "86400", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_scrub_max_keys": "100", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_scrub_timeout": "300", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_session_timeout": "300", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_smart_report_timeout": "5", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_stat_smooth_intervals": "6", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-09-21T10:05:05.290 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_stretch_pool_min_size": "2", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_stretch_pool_size": "4", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_subscribe_interval": "86400.000000", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_sync_debug": "false", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_sync_max_payload_keys": "2000", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_sync_max_payload_size": "1048576", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_sync_provider_kill_at": "0", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_sync_requester_kill_at": "0", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_sync_timeout": "60.000000", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_target_pg_per_osd": "100", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_tick_interval": "5", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_timecheck_interval": "300.000000", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-09-21T10:05:05.291 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_use_min_delay_socket": "false", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_older_version_delay": "604800", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_filestore_osds": "true", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_misplaced": "false", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_older_version": "true", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_pool_no_app": "true", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-09-21T10:05:05.292 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_on_too_few_osds": "false", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "monmap": "", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-09-21T10:05:05.293 INFO:teuthology.orchestra.run.smithi032.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "motr_tracing_enabled": "false", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_op_threads": "3", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_buffer_size": "131072", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_cm": "false", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_device_name": "", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_dscp": "96", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_gid_idx": "0", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_local_gid": "", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_polling_us": "1000", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_port_num": "1", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-09-21T10:05:05.294 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_roce_ver": "1", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_send_buffers": "1024", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_sl": "3", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_support_srq": "true", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_rdma_type": "ib", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_async_reap_threshold": "5", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_before_connect": "false", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_ipv4": "true", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_ipv6": "false", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_msgr1": "true", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_msgr2": "true", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_port_max": "7568", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_port_min": "6800", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_prefer_ipv4": "false", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_retry_count": "3", 2024-09-21T10:05:05.295 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_bind_retry_delay": "5", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_blackhole_client": "false", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_blackhole_mds": "false", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_blackhole_mgr": "false", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_blackhole_mon": "false", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_blackhole_osd": "false", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_client_mode": "crc secure", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_cluster_mode": "crc secure", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_cluster_type": "", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_compress_secure": "false", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_connection_idle_timeout": "900", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_connection_ready_timeout": "10", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_crc_data": "true", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_crc_header": "true", 2024-09-21T10:05:05.296 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_die_on_bad_msg": "false", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_die_on_bug": "true", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_die_on_old_message": "true", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_die_on_skipped_message": "false", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_die_on_unhandled_msg": "false", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_coremask": "0xF", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_devs_allowlist": "", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_enable_tso": "true", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_hugepages": "", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_hw_flow_control": "true", 2024-09-21T10:05:05.297 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_lro": "true", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_memory_channel": "4", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_pmd": "", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_port_id": "0", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dump_corrupt_message_level": "1", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_dump_on_send": "false", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_initial_backoff": "0.200000", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_inject_delay_max": "1.000000", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_inject_delay_probability": "0.000000", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_inject_delay_type": "", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_inject_internal_delays": "0.000000", 2024-09-21T10:05:05.298 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_inject_network_congestion": "0", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_inject_socket_failures": "0", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_learn_addr_from_peer": "true", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_max_accept_failures": "4", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_max_backoff": "15.000000", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_mon_client_mode": "secure crc", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_mon_cluster_mode": "secure crc", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_mon_service_mode": "secure crc", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_osd_compress_min_size": "1024", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_osd_compress_mode": "none", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_osd_compression_algorithm": "snappy", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_pq_min_cost": "65536", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_public_type": "", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_service_mode": "crc secure", 2024-09-21T10:05:05.299 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_tcp_listen_backlog": "512", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_tcp_nodelay": "true", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_tcp_rcvbuf": "0", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_time_events_min_wait_interval": "1000", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "ms_type": "async+posix", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "no_config_file": "false", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "nvmeof_mon_client_tick_period": "2", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "objecter_completion_locks_per_session": "32", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "objecter_debug_inject_relock_delay": "false", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "objecter_inflight_op_bytes": "104857600", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "objecter_inflight_ops": "1024", 2024-09-21T10:05:05.300 INFO:teuthology.orchestra.run.smithi032.stdout: "objecter_inject_no_watch_ping": "false", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "objecter_tick_interval": "5.000000", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "objecter_timeout": "10.000000", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "objectstore_blackhole": "false", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "openssl_engine_opts": "", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_agent_delay_time": "5.000000", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_agent_hist_halflife": "1000", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_agent_max_low_ops": "2", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_agent_max_ops": "4", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_agent_quantize_effort": "0.100000", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_agent_slop": "0.020000", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-09-21T10:05:05.301 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_allow_recovery_below_min_size": "true", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_asio_thread_count": "2", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_async_recovery_min_cost": "100", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_auto_mark_unfound_lost": "false", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_backfill_retry_interval": "30.000000", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_backfill_scan_max": "512", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_backfill_scan_min": "64", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_backoff_on_degraded": "false", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_backoff_on_peering": "false", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_backoff_on_unfound": "true", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_beacon_report_interval": "300", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_bench_duration": "30", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_bench_max_block_size": "67108864", 2024-09-21T10:05:05.302 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_bench_small_size_max_iops": "100", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_blkin_trace_all": "false", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_blocked_scrub_grace_period": "120", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_check_for_log_corruption": "false", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_class_update_on_start": "true", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_client_message_cap": "256", 2024-09-21T10:05:05.303 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_client_message_size_cap": "524288000", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_client_op_priority": "63", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_client_watch_timeout": "30", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_command_max_records": "256", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_compact_on_start": "false", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_crush_chooseleaf_type": "1", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_crush_initial_weight": "-1.000000", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_crush_update_on_start": "true", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_crush_update_weight_set": "true", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_drop_ping_duration": "0", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-09-21T10:05:05.304 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_feed_pullee": "-1", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_misdirected_ops": "false", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_no_acting_change": "false", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_no_purge_strays": "false", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_op_order": "false", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_pg_log_writeout": "false", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_pretend_recovery_active": "false", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-09-21T10:05:05.305 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_shutdown": "false", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_trim_objects": "false", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_verify_cached_snaps": "false", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_verify_missing_on_start": "false", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_verify_snaps": "false", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_deep_scrub_keys": "1024", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-09-21T10:05:05.306 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_deep_scrub_stride": "524288", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_default_data_pool_replay_window": "45", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_default_notify_timeout": "30", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_delete_sleep": "0.000000", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_discard_disconnected_ops": "true", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_ec_partial_reads": "true", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_enable_op_tracker": "true", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_extblkdev_plugins": "vdo", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-09-21T10:05:05.307 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_fast_info": "true", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_fast_shutdown": "true", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_fast_shutdown_timeout": "15", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_function_tracing": "false", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_heartbeat_grace": "20", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_heartbeat_interval": "6", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_heartbeat_min_peers": "10", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_heartbeat_min_size": "2000", 2024-09-21T10:05:05.308 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_heartbeat_stale": "600", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_hit_set_max_size": "100000", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_hit_set_min_size": "1000", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_journal_flush_on_shutdown": "true", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_journal_size": "5120", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_kill_backfill_at": "0", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_loop_before_reset_tphandle": "64", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_map_cache_size": "50", 2024-09-21T10:05:05.309 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_map_dedup": "true", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_map_message_max": "40", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_map_message_max_bytes": "10485760", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_map_share_max_epochs": "40", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_attr_name_len": "100", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_attr_size": "0", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_backfills": "1", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_markdown_count": "5", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_markdown_period": "600", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_object_name_len": "2048", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_object_namespace_len": "256", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_object_size": "134217728", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_omap_entries_per_request": "1024", 2024-09-21T10:05:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_pg_blocked_by": "16", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_pg_log_entries": "10000", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_pgls": "1024", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_push_cost": "8388608", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_push_objects": "10", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_scrubs": "3", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_trimming_pgs": "2", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_write_op_reply_len": "64", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_max_write_size": "90", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-09-21T10:05:05.311 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_override_recovery_settings": "false", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_profile": "balanced", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-09-21T10:05:05.312 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mclock_skip_benchmark": "false", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_memory_base": "805306368", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_memory_cache_min": "134217728", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_memory_target": "4294967296", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_memory_target_autotune": "false", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_min_pg_log_entries": "250", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_min_recovery_priority": "0", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mon_heartbeat_interval": "30", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mon_report_interval": "5", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mon_report_max_in_flight": "2", 2024-09-21T10:05:05.313 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_num_cache_shards": "32", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_num_op_tracker_shard": "32", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_numa_auto_affinity": "true", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_numa_node": "-1", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_numa_prefer_iface": "true", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_objecter_finishers": "1", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_objectstore": "bluestore", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_objectstore_fuse": "false", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_objectstore_ideal_list_max": "64", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_objectstore_tracing": "false", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_complaint_time": "30.000000", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_history_duration": "600", 2024-09-21T10:05:05.314 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_history_size": "20", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_history_slow_op_size": "20", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_log_threshold": "5", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_num_shards": "0", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_num_shards_hdd": "5", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_num_shards_ssd": "8", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_num_threads_per_shard": "0", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_pq_min_cost": "65536", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_queue": "mclock_scheduler", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_queue_cut_off": "high", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_thread_suicide_timeout": "150", 2024-09-21T10:05:05.315 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_op_thread_timeout": "15", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_open_classes_on_start": "true", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_os_flags": "0", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_peering_op_priority": "255", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_delete_cost": "1048576", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_delete_priority": "5", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_log_dups_tracked": "3000", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_log_trim_max": "10000", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_log_trim_min": "100", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-09-21T10:05:05.316 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_object_context_cache_count": "64", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_crimson": "false", 2024-09-21T10:05:05.317 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_crush_rule": "-1", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_ec_fast_read": "false", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_flag_bulk": "false", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_flag_nodelete": "false", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_flags": "0", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_min_size": "0", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_pg_num": "32", 2024-09-21T10:05:05.318 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_pgp_num": "0", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_read_ratio": "70", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_size": "3", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_default_type": "replicated", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_pool_use_gmt_hitset": "true", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_push_per_object_cost": "1000", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_read_ec_check_for_errors": "false", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recover_clone_overlap": "true", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recover_clone_overlap_limit": "10", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_cost": "20971520", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_delay_start": "0.000000", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_max_active": "0", 2024-09-21T10:05:05.319 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_max_active_hdd": "3", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_max_active_ssd": "10", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_max_chunk": "8388608", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_max_single_start": "1", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_op_priority": "3", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_op_warn_multiple": "16", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_priority": "5", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_retry_interval": "30.000000", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_sleep": "0.000000", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_repair_during_recovery": "false", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_requested_scrub_priority": "5", 2024-09-21T10:05:05.320 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_rollback_to_cluster_snap": "", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_auto_repair": "false", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_begin_hour": "0", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_begin_week_day": "0", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_chunk_max": "15", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_chunk_min": "5", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_cost": "52428800", 2024-09-21T10:05:05.321 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_during_recovery": "false", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_end_hour": "0", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_end_week_day": "0", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_event_cost": "4096", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_invalid_stats": "true", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_load_threshold": "0.500000", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_max_interval": "604800.000000", 2024-09-21T10:05:05.322 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_max_preemptions": "5", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_min_interval": "86400.000000", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_priority": "5", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_retry_after_noscrub": "60", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_retry_delay": "30", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_retry_new_interval": "10", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_retry_pg_state": "60", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_retry_trimming": "10", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_scrub_sleep": "0.000000", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-09-21T10:05:05.323 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_shutdown_pgref_assert": "false", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_skip_data_digest": "false", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_smart_report_timeout": "5", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_snap_trim_cost": "1048576", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_snap_trim_priority": "5", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_snap_trim_sleep": "0.000000", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-09-21T10:05:05.324 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_stats_update_period_scrubbing": "15", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_target_transaction_size": "30", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_default_cache_mode": "writeback", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-09-21T10:05:05.325 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_tracing": "false", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_use_stale_snap": "false", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "osdc_blkin_trace_all": "false", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_kill_at": "0", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_max_join_drift": "10", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_min": "500", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_min_wait": "0.050000", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_propose_interval": "1.000000", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_service_trim_max": "500", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_service_trim_max_multiplier": "20", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_service_trim_min": "250", 2024-09-21T10:05:05.326 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_stash_full_interval": "25", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_trim_max": "500", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "paxos_trim_min": "250", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "perf": "true", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "pid_file": "", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "public_addr": "-", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "public_addrv": "", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "public_bind_addr": "-", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "public_network": "172.21.0.0/20", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "public_network_interface": "", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "qat_compressor_busy_polling": "false", 2024-09-21T10:05:05.327 INFO:teuthology.orchestra.run.smithi032.stdout: "qat_compressor_enabled": "false", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "qat_compressor_session_max_number": "256", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rados_mon_op_timeout": "0", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rados_osd_op_timeout": "0", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rados_tracing": "false", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_atime_update_interval": "60", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_balance_parent_reads": "false", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_balance_snap_reads": "false", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_blkin_trace_all": "false", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_blocklist_expire_seconds": "0", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_blocklist_on_break_lock": "true", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_cache": "true", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_cache_block_writes_upfront": "false", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_cache_max_dirty": "25165824", 2024-09-21T10:05:05.328 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_cache_max_dirty_object": "0", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_cache_policy": "writearound", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_cache_size": "33554432", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_cache_target_dirty": "16777216", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_clone_copy_on_read": "false", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_compression_hint": "none", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_concurrent_management_ops": "10", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_clone_format": "auto", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_data_pool": "", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_features": "61", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_format": "2", 2024-09-21T10:05:05.329 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_map_options": "", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_order": "22", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_pool": "rbd", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_stripe_count": "0", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_default_stripe_unit": "0", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_disable_zero_copy_writes": "true", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_discard_granularity_bytes": "65536", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_enable_alloc_hint": "true", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_io_scheduler": "simple", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_commit_age": "5.000000", 2024-09-21T10:05:05.330 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_object_flush_interval": "0", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_order": "24", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_pool": "", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_journal_splay_width": "4", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_localize_parent_reads": "false", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_localize_snap_reads": "false", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-09-21T10:05:05.331 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_image_policy_type": "simple", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_memory_autotune": "true", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_memory_base": "805306368", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-09-21T10:05:05.332 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_memory_target": "4294967296", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirroring_delete_delay": "0", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirroring_replay_delay": "0", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_move_to_trash_on_remove": "false", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_mtime_update_interval": "60", 2024-09-21T10:05:05.333 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_non_blocking_aio": "true", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_op_thread_timeout": "60", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_op_threads": "1", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_parent_cache_enabled": "false", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_persistent_cache_mode": "disabled", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_persistent_cache_path": "/tmp", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_persistent_cache_size": "1073741824", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_plugins": "", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_bps_burst": "0", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_bps_limit": "0", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_exclude_ops": "0", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_iops_burst": "0", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-09-21T10:05:05.334 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_iops_limit": "0", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_read_bps_burst": "0", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_read_bps_limit": "0", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_read_iops_burst": "0", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_read_iops_limit": "0", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_schedule_tick_min": "50", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_write_bps_burst": "0", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_write_bps_limit": "0", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_write_iops_burst": "0", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_qos_write_iops_limit": "0", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_quiesce_notification_attempts": "10", 2024-09-21T10:05:05.335 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_read_from_replica_policy": "default", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_readahead_max_bytes": "524288", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_readahead_trigger_requests": "10", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_request_timed_out_seconds": "30", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_skip_partial_discard": "true", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_tracing": "false", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_validate_names": "true", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rbd_validate_pool": "true", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "restapi_base_url": "", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "restapi_log_level": "", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-09-21T10:05:05.336 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_account_default_quota_max_size": "-1", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_acl_grants_max_num": "100", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_admin_entry": "admin", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_asio_assert_yielding": "false", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_backend_store": "rados", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_barbican_url": "", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_beast_enable_async": "true", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_bucket_counters_cache": "false", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_bucket_index_max_aio": "128", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-09-21T10:05:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_bucket_quota_ttl": "600", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_cache_enabled": "true", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_cache_expiry_interval": "900", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_cache_lru_size": "25000", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_config_store": "rados", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_content_length_compat": "false", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_copy_obj_progress": "true", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_cors_rules_max_num": "100", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_cross_domain_policy": "", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_default_encryption_key": "", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_kmip_addr": "", 2024-09-21T10:05:05.338 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_kmip_ca_path": "", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_kmip_client_cert": "", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_kmip_client_key": "", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_kmip_password": "", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_kmip_username": "", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_require_ssl": "true", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-09-21T10:05:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_suppress_logs": "true", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_addr": "", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_auth": "token", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_namespace": "", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_prefix": "", 2024-09-21T10:05:05.340 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_token_file": "", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_curl_buffersize": "524288", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_curl_low_speed_limit": "1024", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_curl_low_speed_time": "300", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_curl_tcp_keepalive": "0", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-09-21T10:05:05.341 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d3n_l1_fadvise": "4", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d4n_l1_fadvise": "4", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-09-21T10:05:05.342 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_data_log_changes_size": "1000", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_data_log_num_shards": "128", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_data_log_window": "30", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_data_notify_interval_msec": "0", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_data_sync_poll_interval": "20", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_data_sync_spawn_window": "20", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_default_data_log_backing": "fifo", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_default_region_info_oid": "default.region", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-09-21T10:05:05.343 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_defer_to_bucket_acls": "", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_disable_s3select": "false", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_data_res": "500.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-09-21T10:05:05.344 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dns_name": "", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dns_s3website_name": "", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dynamic_resharding": "true", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_enable_gc_threads": "true", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_enable_lc_threads": "true", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_enable_mdsearch": "true", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_enable_ops_log": "false", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_enable_quota_threads": "true", 2024-09-21T10:05:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_enable_static_website": "false", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_enable_usage_log": "false", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_enforce_swift_acls": "true", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_exit_timeout_secs": "120", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_expose_bucket": "false", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_extended_http_attrs": "", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_filter": "none", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_frontends": "beast port=7480", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_gc_max_concurrent_io": "10", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_gc_max_deferred": "50", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_gc_max_objs": "32", 2024-09-21T10:05:05.346 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_gc_max_queue_size": "134213632", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_gc_max_trim_chunk": "16", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_gc_obj_min_wait": "7200", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_gc_processor_max_time": "3600", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_gc_processor_period": "3600", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_get_obj_window_size": "16777216", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_healthcheck_disabling_path": "", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ignore_get_invalid_range": "false", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_init_timeout": "300", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_kafka_connection_idle": "300", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_kafka_message_timeout": "5000", 2024-09-21T10:05:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_kafka_sleep_timeout": "10", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_admin_domain": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_admin_password": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_admin_password_path": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_admin_project": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_admin_tenant": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_admin_token": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_admin_token_path": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_admin_user": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_barbican_domain": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_barbican_password": "", 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_barbican_project": "", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_barbican_tenant": "", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_barbican_user": "", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_implicit_tenants": "false", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_service_token_enabled": "false", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_token_cache_size": "10000", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_url": "", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_keystone_verify_ssl": "true", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lc_debug_interval": "-1", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lc_lock_max_time": "90", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lc_max_objs": "32", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lc_max_rules": "1000", 2024-09-21T10:05:05.349 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lc_max_worker": "3", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lc_max_wp_worker": "3", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lc_thread_delay": "0", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ldap_dnattr": "uid", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ldap_searchfilter": "", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ldap_uri": "", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lfuda_sync_frequency": "60", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_log_http_headers": "", 2024-09-21T10:05:05.350 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_log_nonexistent_bucket": "false", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_log_object_name_utc": "false", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_attr_name_len": "0", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_attr_size": "0", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_attrs_num_in_req": "0", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_chunk_size": "4194304", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_concurrent_requests": "1024", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_dynamic_shards": "1999", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_listing_results": "5000", 2024-09-21T10:05:05.351 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_notify_retries": "10", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_objs_per_shard": "100000", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_put_param_size": "1048576", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_put_size": "5368709120", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_max_slo_entries": "1000", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_md_log_max_shards": "64", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_md_notify_interval_msec": "200", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_meta_sync_poll_interval": "20", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_meta_sync_spawn_window": "20", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_mp_lock_max_time": "600", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_multipart_min_part_size": "5242880", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-09-21T10:05:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_fhcache_size": "2017", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_lru_lanes": "5", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_max_gc": "300", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_run_gc_threads": "false", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_run_lc_threads": "false", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_run_quota_threads": "false", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_run_sync_thread": "false", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_num_async_rados_threads": "32", 2024-09-21T10:05:05.353 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_num_control_oids": "8", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_numa_node": "-1", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_obj_stripe_size": "4194304", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_objexp_chunk_size": "100", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_objexp_gc_interval": "600", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_objexp_hints_num_shards": "127", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_op_thread_timeout": "600", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_op_tracing": "false", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_opa_token": "", 2024-09-21T10:05:05.354 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_opa_url": "", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_opa_verify_ssl": "true", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ops_log_rados": "false", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_ops_log_socket_path": "", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_period_push_interval": "2.000000", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_period_push_interval_max": "30.000000", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_period_root_pool": ".rgw.root", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-09-21T10:05:05.355 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_posix_cache_lanes": "3", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_posix_cache_max_buckets": "100", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_posix_cache_partitions": "3", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_print_continue": "true", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_print_prohibited_content_length": "false", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_rados_tracing": "false", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_realm": "", 2024-09-21T10:05:05.356 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_realm_id": "", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_region": "", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_region_root_pool": ".rgw.root", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_relaxed_region_enforcement": "false", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_request_uri": "", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_reshard_batch_size": "64", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_reshard_debug_interval": "-1", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_reshard_max_aio": "128", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_reshard_num_logs": "16", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_reshard_progress_judge_interval": "120", 2024-09-21T10:05:05.357 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_reshard_progress_judge_ratio": "0.500000", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_reshard_thread_interval": "600", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_reshardlog_threshold": "30000", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_resolve_cname": "false", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_rest_getusage_op_compat": "false", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_run_sync_thread": "true", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_s3_auth_use_keystone": "false", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_s3_auth_use_ldap": "false", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_s3_auth_use_rados": "true", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_s3_auth_use_sts": "false", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-09-21T10:05:05.358 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_s3_success_create_obj_status": "0", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_scheduler_type": "throttler", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_script_uri": "", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_service_provider_name": "", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sts_client_id": "", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sts_client_secret": "", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sts_entry": "sts", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sts_key": "sts", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sts_max_session_duration": "43200", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sts_min_session_duration": "900", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sts_token_introspection_url": "", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_account_in_url": "false", 2024-09-21T10:05:05.359 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_auth_entry": "auth", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_auth_url": "", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_custom_header": "", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_enforce_content_length": "false", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_need_stats": "true", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_tenant_name": "", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_token_expiration": "86400", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_url": "", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_url_prefix": "swift", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_swift_versioning_enabled": "false", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-09-21T10:05:05.360 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_lease_period": "120", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_log_trim_interval": "1200", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_obj_etag_verify": "false", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_trace_history_size": "4096", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_thread_pool_size": "512", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_topic_persistency_max_retries": "0", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-09-21T10:05:05.361 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_topic_require_publish_policy": "false", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_torrent_comment": "", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_torrent_createby": "", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_torrent_encoding": "", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_torrent_flag": "false", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_torrent_max_size": "5368709120", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_torrent_origin": "", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_torrent_sha_unit": "524288", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_torrent_tracker": "", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_trust_forwarded_https": "false", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_usage_log_tick_interval": "30", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_usage_max_shards": "32", 2024-09-21T10:05:05.362 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_usage_max_user_shards": "1", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_use_opa_authz": "false", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_counters_cache": "false", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_counters_cache_size": "10000", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_default_quota_max_size": "-1", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_max_buckets": "1000", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_policies_max_num": "100", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_quota_sync_interval": "86400", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_user_unique_email": "true", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_verify_ssl": "true", 2024-09-21T10:05:05.363 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_website_routing_rules_max_num": "50", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_zone": "", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_zone_id": "", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_zonegroup": "", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_zonegroup_id": "", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rgwlc_auto_session_clear": "true", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rgwlc_skip_bucket_step": "false", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_block_size": "4096", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_cache_shard_bits": "4", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_cache_size": "536870912", 2024-09-21T10:05:05.364 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_cache_type": "binned_lru", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_collect_compaction_stats": "false", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_collect_extended_stats": "false", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_collect_memory_stats": "false", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_index_type": "binary_search", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_log_to_ceph_log": "true", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_metadata_block_size": "4096", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_partition_filters": "false", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_perf": "false", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-09-21T10:05:05.365 INFO:teuthology.orchestra.run.smithi032.stdout: "rotating_keys_renewal_timeout": "10", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "run_dir": "/var/run/ceph", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_block_create": "true", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_cache_lru_size": "67108864", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_cbjournal_size": "5368709120", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_data_delta_based_overwrite": "0", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_default_max_object_size": "16777216", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_device_size": "53687091200", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_full_integrity_check": "false", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_journal_batch_capacity": "16", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-09-21T10:05:05.366 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_journal_iodepth_limit": "5", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_main_device_type": "SSD", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_max_concurrent_transactions": "8", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_max_data_allocation_size": "32768", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "seastore_segment_size": "67108864", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "set_keepcaps": "false", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "setgroup": "ceph", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "setuser": "ceph", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "setuser_match_path": "", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "target_max_misplaced_ratio": "0.050000", 2024-09-21T10:05:05.367 INFO:teuthology.orchestra.run.smithi032.stdout: "thp": "false", 2024-09-21T10:05:05.368 INFO:teuthology.orchestra.run.smithi032.stdout: "threadpool_default_timeout": "60", 2024-09-21T10:05:05.368 INFO:teuthology.orchestra.run.smithi032.stdout: "threadpool_empty_queue_max_wait": "2", 2024-09-21T10:05:05.368 INFO:teuthology.orchestra.run.smithi032.stdout: "throttler_perf_counter": "true", 2024-09-21T10:05:05.368 INFO:teuthology.orchestra.run.smithi032.stdout: "tmp_dir": "/tmp", 2024-09-21T10:05:05.368 INFO:teuthology.orchestra.run.smithi032.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-09-21T10:05:05.368 INFO:teuthology.orchestra.run.smithi032.stdout: "uadk_compressor_enabled": "false", 2024-09-21T10:05:05.368 INFO:teuthology.orchestra.run.smithi032.stdout: "uadk_wd_sync_ctx_num": "2" 2024-09-21T10:05:05.368 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:05:05.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:05 smithi045 ceph-mon[95976]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:06.279 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-09-21T10:05:06.484 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:07 smithi032 ceph-mon[101299]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:07.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:07 smithi032 ceph-mon[105748]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:07 smithi045 ceph-mon[95976]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:08.279 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a: {} 2024-09-21T10:05:08.279 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b: {} 2024-09-21T10:05:08.279 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c: {} 2024-09-21T10:05:08.279 INFO:teuthology.orchestra.run.smithi032.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:05:08.280 INFO:teuthology.orchestra.run.smithi032.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:05:08.280 INFO:teuthology.orchestra.run.smithi032.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:05:08.925 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-09-21T10:05:08.925 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-21T10:05:08.926 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:05:08.928 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:05:08.928 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-21T10:05:08.928 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_0 16 2024-09-21T10:05:08.928 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-09-21T10:05:08.928 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:05:08.929 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:05:08.929 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-09-21T10:05:08.929 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:05:08.929 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:05:08.929 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-09-21T10:05:08.929 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-09-21T10:05:08.929 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:08.930 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-09-21T10:05:08.930 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:05:08.930 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-09-21T10:05:08.930 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:05:08.931 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-09-21T10:05:08.931 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:05:08.959 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:08.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:09.061 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:09.095 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:09.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:09.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:09 smithi032 ceph-mon[101299]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:09.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2524603470' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:05:09.185 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:09 smithi032 ceph-mon[105748]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:09.185 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2524603470' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:05:09.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:09.221 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:09.221 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:09.224 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:09.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:09.271 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:09.326 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:09.353 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:09.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:09.455 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:09.482 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:09.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:09.503 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:09.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:09 smithi045 ceph-mon[95976]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:09.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2524603470' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:05:09.584 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:09.607 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:09.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:09.708 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:09.732 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:09.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:09.806 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:05:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:05:09] "GET /metrics HTTP/1.1" 200 33927 "" "Prometheus/2.51.0" 2024-09-21T10:05:09.834 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:09.856 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:09.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:09.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:09.985 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:10.086 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:10.112 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:10.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:10.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:10.240 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:10.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:10.367 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:10.468 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:10.492 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:10.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:10.594 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:10.618 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:10.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:10.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:10.746 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:10.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:10.848 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:10.874 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:10.976 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:11.002 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:11.104 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:11.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:11.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:11.232 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:11.258 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:11.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:11.360 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:11.386 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:11.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:11.410 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:11 smithi032 ceph-mon[105748]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:11.411 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:11 smithi032 ceph-mon[101299]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:11.487 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:11.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.512 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:11.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:11.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:11 smithi045 ceph-mon[95976]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:11.614 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:11.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.642 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:11.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:11.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:11.767 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:11.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:11.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:11.892 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:11.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:11.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:12.021 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:12.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:12.123 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:12.149 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:12.251 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:12.277 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:12.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:12.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:12.406 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:12.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:12.507 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:12.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:12.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:12.636 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:12.659 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:12.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:12.761 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:12.787 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:12.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:12.889 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:12.911 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:12.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:13.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:13.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:13.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:13.164 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:13.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:13.266 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:13.288 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:13.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:13.294 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:13 smithi032 ceph-mon[105748]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:13.294 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:13 smithi032 ceph-mon[101299]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:13.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:13.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:13.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:13.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:13 smithi045 ceph-mon[95976]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:13.539 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:13.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:13.663 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:13.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:13.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:13.787 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:13.889 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:13.912 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:13.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:14.014 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:14.037 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:14.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:14.139 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:14.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:14.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:14.197 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:05:14.263 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:14.285 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:14.387 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:14.408 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:14.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:14.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:14 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/902843518' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:14 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:14 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/902843518' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:14 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:14.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:14.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:14.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:14 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/902843518' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:14.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:14 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:14.635 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.659 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:14.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:14.759 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:14.782 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:14.884 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:14.906 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:15.007 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:15.030 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:15.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:15.100 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:05:15.100 INFO:teuthology.orchestra.run.smithi032.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:05:15.100 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:05:15.132 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:15.155 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:15.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:15.257 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:15.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.282 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:15.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:15.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:15.405 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:15.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:15.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:15 smithi032 ceph-mon[101299]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:15.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:15 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:15.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:15 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:05:15.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:15 smithi032 ceph-mon[101299]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T10:05:15.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:15 smithi032 ceph-mon[105748]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:15.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:15 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:15.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:15 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:05:15.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:15 smithi032 ceph-mon[105748]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T10:05:15.507 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:15.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:15 smithi045 ceph-mon[95976]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:15 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:15 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:05:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:15 smithi045 ceph-mon[95976]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T10:05:15.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:15.636 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:15.659 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:15.761 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:15.782 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:15.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:15.884 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.907 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:15.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:16.008 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:16.071 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:16.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:16.173 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:16.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:16.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:16.303 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:16.325 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:16.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:16.427 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:16.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:16.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:16.553 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:05:16.553 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:05:16.553 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 0 2024-09-21T10:05:16.554 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:05:16.556 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:16.585 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:16.688 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:16.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.716 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:16.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:16.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.846 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:16.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:16.893 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:05:16.948 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:16.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:16.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:17.077 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:17.099 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:17.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:17.176 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:05:17.201 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:17.227 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:17.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:17.329 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:17.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:17.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:17 smithi032 ceph-mon[101299]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:17 smithi032 ceph-mon[105748]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:17.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:17.480 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:17.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:17 smithi045 ceph-mon[95976]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:17.582 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:17.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:17.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:17.710 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:17.737 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:17.839 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:17.861 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:17.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:17.963 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:17.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:17.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:18.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:18.114 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:18.200 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_0' created 2024-09-21T10:05:18.216 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:18.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:18.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:18.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:18.368 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:18.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:18.378 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:18 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1337235776' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-21T10:05:18.378 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:18 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1337235776' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-21T10:05:18.469 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:18.495 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:18.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:18 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1337235776' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-21T10:05:18.597 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:18.620 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:18.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:18.666 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:18.671 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:18.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:18.744 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:18.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:18.845 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:18.872 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:18.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:19.001 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:19.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:19.102 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:19.127 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:19.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:19.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-21T10:05:19.229 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:19.255 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:19.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:19.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:19.369 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[101299]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:19.369 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1337235776' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-21T10:05:19.369 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[101299]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T10:05:19.369 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:19.369 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:19.370 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[105748]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:19.370 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1337235776' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-21T10:05:19.370 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[105748]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T10:05:19.370 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:19.370 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:19.384 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:19.486 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:19.512 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:19.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:19.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:19 smithi045 ceph-mon[95976]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1337235776' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-21T10:05:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:19 smithi045 ceph-mon[95976]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T10:05:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:05:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:19.613 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:19.636 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:19.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:19.704 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:05:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:05:19] "GET /metrics HTTP/1.1" 200 36278 "" "Prometheus/2.51.0" 2024-09-21T10:05:19.738 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:19.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:19.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:19.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:19.892 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:19.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:19.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:20.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:20.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:20.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:20.148 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:20.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:20.250 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:20.273 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:20.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:20.375 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:20.398 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:20.500 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:20.523 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:20.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:20.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:20 smithi045 ceph-mon[95976]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T10:05:20.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:20.626 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:20 smithi032 ceph-mon[101299]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T10:05:20.627 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:20 smithi032 ceph-mon[105748]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T10:05:20.649 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:20.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:20.750 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:20.772 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:20.874 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:20.901 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:20.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:21.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:21.011 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:05:21.011 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:05:21.011 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-21T10:05:21.011 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:05:21.012 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:05:21.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-21T10:05:21.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:21.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:21.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:21.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:21.258 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:21.284 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:21.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:21.364 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:21 smithi032 ceph-mon[105748]: pgmap v297: 89 pgs: 7 creating+peering, 8 unknown, 74 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:21.365 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:21 smithi032 ceph-mon[105748]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:21.365 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:21 smithi032 ceph-mon[105748]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T10:05:21.365 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:21 smithi032 ceph-mon[101299]: pgmap v297: 89 pgs: 7 creating+peering, 8 unknown, 74 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:21.365 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:21 smithi032 ceph-mon[101299]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:21.365 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:21 smithi032 ceph-mon[101299]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T10:05:21.370 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:21.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:21.412 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:21.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:21.514 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:21.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:21 smithi045 ceph-mon[95976]: pgmap v297: 89 pgs: 7 creating+peering, 8 unknown, 74 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:21 smithi045 ceph-mon[95976]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:21 smithi045 ceph-mon[95976]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T10:05:21.536 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:21.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:21.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:21.664 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:21.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:21.792 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:21.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:21.917 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:21.940 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:21.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:22.041 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:22.068 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:22.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:22.170 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:22.196 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:22.298 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:22.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:22.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:22.422 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:22.445 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:22.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:22.535 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:05:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=cleanup t=2024-09-21T10:05:22.126486085Z level=info msg="Completed cleanup jobs" duration=2.821475ms 2024-09-21T10:05:22.535 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:05:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=sqlstore.transactions t=2024-09-21T10:05:22.133076879Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T10:05:22.535 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:05:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=plugins.update.checker t=2024-09-21T10:05:22.256847186Z level=info msg="Update check succeeded" duration=59.049919ms 2024-09-21T10:05:22.547 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:22.573 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:22.675 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:22.697 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:22.799 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:22.825 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:22.826 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:22.827 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:22.926 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:05:22.926 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:05:22.926 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:05:22.926 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:05:22.926 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:05:22.926 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:22.926 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:22.926 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:22.927 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:22.927 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:22.927 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:05:22.927 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:05:22.927 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:05:22.927 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:22.927 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:22.927 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:22.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:22.954 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:22.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:23.056 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:23.079 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:23.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:23.148 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:23.181 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:23.207 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:23.309 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:23.331 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:23.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:23.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:23.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:23.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:23.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:23 smithi032 ceph-mon[101299]: pgmap v299: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:23.487 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1100439385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:23.487 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:23 smithi032 ceph-mon[105748]: pgmap v299: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:23.488 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1100439385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:23 smithi045 ceph-mon[95976]: pgmap v299: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1100439385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:23.557 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:23.579 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:23.681 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:23.707 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:23.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:23.809 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:23.835 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:23.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:23.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:23.960 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:23.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:24.062 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:24.084 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:24.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:24.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:24.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:24.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:24.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:24.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:24.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:24.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:24.465 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:24.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:24.567 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:24.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:24.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:24.695 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:24.717 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:24.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:24.841 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:24.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:24.943 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:24.968 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:24.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:25.069 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:25.092 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:25.194 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:25.220 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:25.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:25.226 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:05:25.322 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:25.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:25.449 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:25.472 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:25.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:25.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:25 smithi045 ceph-mon[95976]: pgmap v300: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:25.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1204916542' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:25.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:25 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:25.573 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:25.595 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:25 smithi032 ceph-mon[105748]: pgmap v300: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:25.595 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1204916542' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:25.595 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:25 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:25.595 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:25 smithi032 ceph-mon[101299]: pgmap v300: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:25.595 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1204916542' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:25.595 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:25 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:25.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:25.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:25.725 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:25.826 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:25.853 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:25.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:25.981 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:25.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:26.082 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:26.105 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:26.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:26.207 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:26.234 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:26.336 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:26.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:26 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:26 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:26 smithi032 ceph-mon[101299]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T10:05:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:26 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:26 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:26 smithi032 ceph-mon[105748]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T10:05:26.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:26.484 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:26.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:26 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:26 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:26 smithi045 ceph-mon[95976]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T10:05:26.585 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:26.612 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:26.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:26.714 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.742 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:26.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:26.794 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:05:26.794 INFO:teuthology.orchestra.run.smithi032.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:05:26.794 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:05:26.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:26.868 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:26.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:26.970 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:26.993 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:26.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:27.094 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:27.121 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:27.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:27.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:27.232 INFO:teuthology.orchestra.run.smithi032.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-21T10:05:27.248 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:27.350 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:27.377 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:27.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:27 smithi032 ceph-mon[105748]: pgmap v302: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:27.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1466843154' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:27.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:27 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:27.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:27 smithi032 ceph-mon[101299]: pgmap v302: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:27.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1466843154' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:27.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:27 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:27.478 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:27.505 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:27.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:27.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:27 smithi045 ceph-mon[95976]: pgmap v302: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1466843154' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:27 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:27.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:27.632 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:27.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:27.734 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:27.760 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:27.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:27.862 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:27.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:27.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:27.921 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:05:27.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:28.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:28.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:28.075 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:05:28.114 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:28.141 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:28.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:28.265 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:28.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:28.366 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:28.392 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:28.493 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:28.500 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:28 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:28.500 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:28 smithi032 ceph-mon[105748]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T10:05:28.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:28 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:28.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:28 smithi032 ceph-mon[101299]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T10:05:28.516 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:28.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:28.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:28 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:28 smithi045 ceph-mon[95976]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T10:05:28.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:28.640 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:28.742 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:28.769 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:28.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:28.870 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:28.897 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:28.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:28.982 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:28.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:29.000 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:29.025 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:29.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:29.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:29.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:29.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:29.251 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:29.278 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:29.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:29.380 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:29.406 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:29.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:29.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:29 smithi032 ceph-mon[101299]: pgmap v304: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:29.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:29 smithi032 ceph-mon[101299]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:29.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:29 smithi032 ceph-mon[105748]: pgmap v304: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:29.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:29 smithi032 ceph-mon[105748]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:29.507 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:29.532 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:29.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:29.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:29 smithi045 ceph-mon[95976]: pgmap v304: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:29.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:29 smithi045 ceph-mon[95976]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:29.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:29.658 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:29.760 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:29.784 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:05:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:05:29] "GET /metrics HTTP/1.1" 200 36278 "" "Prometheus/2.51.0" 2024-09-21T10:05:29.787 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:29.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:29.905 DEBUG:teuthology.orchestra.run.smithi032:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-09-21T10:05:29.908 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:05:29.908 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:05:29.915 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:29.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:29.959 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:hints = 1 2024-09-21T10:05:29.959 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:05:29.959 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object prefix: benchmark_data_smithi032_140417 2024-09-21T10:05:29.960 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:05:29.960 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:05:30.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:30.061 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:30.162 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:30.202 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:30.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:30.304 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:30.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:30.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:30.441 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:30.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:30.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:30.502 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:05:30.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:30.611 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:30.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:30.713 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:30.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.758 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:30.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:30.859 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:30.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.900 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:30.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:30.960 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 1 16 3689 3673 229.549 229.562 0.00355621 0.00434329 2024-09-21T10:05:31.001 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:31.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.039 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:31.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:31.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:31.173 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:31.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:31.274 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:31.308 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:31.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:31.410 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:31.450 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:31.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:31.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:31 smithi045 ceph-mon[95976]: pgmap v305: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:31.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:31.589 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:31.603 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:31 smithi032 ceph-mon[105748]: pgmap v305: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:31.603 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:31 smithi032 ceph-mon[101299]: pgmap v305: 89 pgs: 89 active+clean; 4.6 MiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:05:31.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:31.752 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:31.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:31.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:31.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:31.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:31.960 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 2 16 7381 7365 230.135 230.75 0.0049178 0.00433585 2024-09-21T10:05:31.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:32.021 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:32.123 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:32.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:32.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:32.269 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:32.308 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:32.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:32.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:32.445 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:32.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:32.546 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:32.546 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:32.546 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:32.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:32.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:32.667 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:05:32.668 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 0 1 2024-09-21T10:05:32.684 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:32.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.752 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:32.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:32.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:32.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.898 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:32.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:32.960 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 3 16 10506 10490 218.521 195.312 0.00581019 0.00456754 2024-09-21T10:05:32.980 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:32.999 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:33.053 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:33.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:33.154 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:33.192 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:33.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:33.249 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:05:33.293 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:33.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.329 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:33.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:33.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:33.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:33 smithi032 ceph-mon[105748]: pgmap v306: 89 pgs: 89 active+clean; 54 MiB data, 353 MiB used, 715 GiB / 715 GiB avail; 6.1 MiB/s wr, 98 op/s 2024-09-21T10:05:33.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:33 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2177590670' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:33.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:33 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:33.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:33 smithi032 ceph-mon[101299]: pgmap v306: 89 pgs: 89 active+clean; 54 MiB data, 353 MiB used, 715 GiB / 715 GiB avail; 6.1 MiB/s wr, 98 op/s 2024-09-21T10:05:33.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:33 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2177590670' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:33.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:33 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:33.466 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:33.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:33 smithi045 ceph-mon[95976]: pgmap v306: 89 pgs: 89 active+clean; 54 MiB data, 353 MiB used, 715 GiB / 715 GiB avail; 6.1 MiB/s wr, 98 op/s 2024-09-21T10:05:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:33 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2177590670' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:33 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:33.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:33.605 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:33.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:33.707 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:33.743 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:33.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:33.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:05:33.845 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:33.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:33.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 4 16 13777 13761 214.995 204.438 0.0019285 0.00464418 2024-09-21T10:05:33.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:34.035 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:34.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:34.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:34.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:34.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:34.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:34.314 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:34.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:34.341 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:34 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:34.341 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:34 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:05:34.341 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:34 smithi032 ceph-mon[105748]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T10:05:34.341 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:34.342 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:34 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:34.342 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:34 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:05:34.342 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:34 smithi032 ceph-mon[101299]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T10:05:34.342 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:34.417 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:34.455 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:34 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:34 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:05:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:34 smithi045 ceph-mon[95976]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T10:05:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:34.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:34.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:34.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:34.694 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:34.731 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:34.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:34.833 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:34.855 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:34.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:34.957 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:34.960 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 5 16 16810 16794 209.906 189.562 0.00456227 0.0047591 2024-09-21T10:05:34.994 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:34.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:35.097 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:35.118 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:35.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:35.220 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:35.257 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:35.359 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:35.396 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:35.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:35.497 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:35.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:35 smithi045 ceph-mon[95976]: pgmap v308: 89 pgs: 89 active+clean; 638 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-21T10:05:35.535 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:35.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:35.641 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:35 smithi032 ceph-mon[105748]: pgmap v308: 89 pgs: 89 active+clean; 638 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-21T10:05:35.644 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:35 smithi032 ceph-mon[101299]: pgmap v308: 89 pgs: 89 active+clean; 638 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-21T10:05:35.659 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:35.710 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:35.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:35.812 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:35.849 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:35.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:35.951 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:35.960 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 6 16 19775 19759 205.804 185.312 0.0055018 0.00485276 2024-09-21T10:05:35.987 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:35.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:36.089 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:36.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.129 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:36.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:36.230 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:36.262 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:36.364 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:36.405 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:36.507 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:36.551 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:36.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:36.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:36.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:36.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:36.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.828 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:36.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:36.929 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:36.960 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 7 16 22643 22627 202.008 179.25 0.00446195 0.00494469 2024-09-21T10:05:36.966 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:36.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:37.068 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:37.105 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:37.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:37.207 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:37.248 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:37.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:37.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:37.392 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:37.495 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:37.496 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:37.526 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:37.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:37 smithi045 ceph-mon[95976]: pgmap v309: 89 pgs: 89 active+clean; 819 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:05:37.548 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:37 smithi032 ceph-mon[101299]: pgmap v309: 89 pgs: 89 active+clean; 819 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:05:37.549 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:05:37.549 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:37.549 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:37 smithi032 ceph-mon[105748]: pgmap v309: 89 pgs: 89 active+clean; 819 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:05:37.584 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:05:37.585 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:37.625 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:05:37.625 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:37.628 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:37.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.660 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:37.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:37.662 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:05:37.662 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:37.688 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:05:37.700 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:05:37.701 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:37.738 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:05:37.738 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:37.761 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:37.773 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:05:37.773 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:37.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.805 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:37.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:37.806 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:05:37.807 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:37.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:37.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:37.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:37.960 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 8 16 25220 25204 196.888 161.062 0.0037472 0.00507295 2024-09-21T10:05:38.059 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:38.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:38.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:38.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:38.238 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:38.340 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:38.376 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:38.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:38.394 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:38 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3152561240' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:38.395 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:38 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3152561240' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:38.478 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:38.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.509 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:38.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:38 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3152561240' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:38.610 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:38.632 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:38.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:38.734 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:38.768 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:38.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:38.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.910 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:38.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:38.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 9 16 27148 27132 188.401 120.5 0.00348585 0.00530254 2024-09-21T10:05:39.011 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:39.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:39.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:39.149 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:39.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.183 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:39.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:39.284 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:39.324 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:39.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:39.352 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:39 smithi032 ceph-mon[105748]: pgmap v310: 89 pgs: 89 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-21T10:05:39.353 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:39 smithi032 ceph-mon[101299]: pgmap v310: 89 pgs: 89 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-21T10:05:39.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:39.448 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:39.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:39.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:39 smithi045 ceph-mon[95976]: pgmap v310: 89 pgs: 89 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-21T10:05:39.550 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:39.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:39.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:39.692 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:39.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:05:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:05:39] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.51.0" 2024-09-21T10:05:39.730 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:39.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:39.832 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:39.868 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:39.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:39.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 10 16 29659 29643 185.253 156.938 0.00454075 0.00539261 2024-09-21T10:05:39.970 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.005 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:40.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:40.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:40.143 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:40.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:40.245 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:40.283 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:40.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:40.385 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:40.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.420 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:40.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:40.521 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:40.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:40.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:40.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:40.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:40.796 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:40.823 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:40.925 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:40.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 11 16 32015 31999 181.797 147.25 0.0042061 0.00549525 2024-09-21T10:05:40.962 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:40.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:41.064 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:41.090 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:41.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:05:41.192 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:41.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.232 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:41.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:41.333 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:41.369 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:41.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:41.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:41 smithi032 ceph-mon[105748]: pgmap v311: 89 pgs: 89 active+clean; 1.5 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-21T10:05:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:41 smithi032 ceph-mon[101299]: pgmap v311: 89 pgs: 89 active+clean; 1.5 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-21T10:05:41.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:41.506 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:41.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:41.607 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:41.643 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:41.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:41.745 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:41.781 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:41.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:41.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:41 smithi045 ceph-mon[95976]: pgmap v311: 89 pgs: 89 active+clean; 1.5 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-21T10:05:41.882 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:41.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:41.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:41.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 12 16 34040 34024 177.192 126.562 0.00417616 0.00562937 2024-09-21T10:05:42.019 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:42.061 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:42.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:42.163 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:42.203 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:42.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:42.305 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:42.330 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:42.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:42.432 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:42.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:42.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:42.612 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:42.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:42.714 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:42.751 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:42.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:42.853 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:42.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:42.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:42.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 13 16 35696 35680 171.523 103.5 0.080397 0.00581213 2024-09-21T10:05:42.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:43.027 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:43.129 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:43.164 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:43.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:43.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:43.308 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:43.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:43.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:43.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:43.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:43.583 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:43.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:43.685 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:43.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:43 smithi032 ceph-mon[105748]: pgmap v312: 89 pgs: 89 active+clean; 1.7 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 168 MiB/s wr, 2.68k op/s 2024-09-21T10:05:43.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:43 smithi032 ceph-mon[101299]: pgmap v312: 89 pgs: 89 active+clean; 1.7 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 168 MiB/s wr, 2.68k op/s 2024-09-21T10:05:43.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.725 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:43.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:43 smithi045 ceph-mon[95976]: pgmap v312: 89 pgs: 89 active+clean; 1.7 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 168 MiB/s wr, 2.68k op/s 2024-09-21T10:05:43.826 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:43.863 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:43.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 14 16 37738 37722 168.386 127.625 0.00209861 0.00593278 2024-09-21T10:05:43.965 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:43.986 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:44.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:44.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.127 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:44.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:44.228 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:44.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:44.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:44.362 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:44.399 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:44.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:44.501 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:44.540 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:44.642 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:44.679 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:44.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:44.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:44.802 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:44.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:44.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:44.941 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:44.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 15 16 39509 39493 164.539 110.688 0.0109244 0.00607247 2024-09-21T10:05:45.043 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:45.081 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:45.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:45.183 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:45.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.211 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:45.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:45.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:45.346 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:45.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:45.448 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:45.483 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:45.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:45.585 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:45.612 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:45.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:45 smithi032 ceph-mon[105748]: pgmap v313: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 181 MiB/s wr, 2.89k op/s 2024-09-21T10:05:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:45 smithi032 ceph-mon[101299]: pgmap v313: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 181 MiB/s wr, 2.89k op/s 2024-09-21T10:05:45.713 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:45.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:45.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:45 smithi045 ceph-mon[95976]: pgmap v313: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 181 MiB/s wr, 2.89k op/s 2024-09-21T10:05:45.851 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:45.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:45.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:45.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 16 16 41269 41253 161.13 110 0.00290807 0.00620086 2024-09-21T10:05:45.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:46.024 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:46.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:46.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:46.163 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:46.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:46.264 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:46.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:46.296 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:46.398 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:46.432 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.0 to 1 (10000) 2024-09-21T10:05:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.443 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:46.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:46.544 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:46.580 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:46.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:46.682 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:46.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:46 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3291888948' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:05:46.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:46 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3291888948' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:05:46.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.720 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:46.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:46 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3291888948' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:05:46.821 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:46.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:46.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:46.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 17 16 43332 43316 159.236 128.938 0.00473545 0.00627528 2024-09-21T10:05:46.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:47.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.011 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:47.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:47.112 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:47.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:47.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:47.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:47.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:47.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:47.387 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:47.422 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:47.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:47.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:47.560 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:47.662 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:47.700 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:47.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:47 smithi032 ceph-mon[105748]: pgmap v314: 89 pgs: 89 active+clean; 2.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-21T10:05:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:47 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3291888948' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:05:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:47 smithi032 ceph-mon[105748]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T10:05:47.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:47 smithi032 ceph-mon[101299]: pgmap v314: 89 pgs: 89 active+clean; 2.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-21T10:05:47.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:47 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3291888948' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:05:47.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:47 smithi032 ceph-mon[101299]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T10:05:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:47 smithi045 ceph-mon[95976]: pgmap v314: 89 pgs: 89 active+clean; 2.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-21T10:05:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:47 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3291888948' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:05:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:47 smithi045 ceph-mon[95976]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T10:05:47.801 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:47.833 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:47.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:47.935 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:47.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 18 16 45450 45434 157.743 132.375 0.00570932 0.00633458 2024-09-21T10:05:47.971 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:47.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:48.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:48.111 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:48.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:48.213 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:48.250 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:48.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:48.268 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:48.295 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:05:48.352 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:48.392 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:48.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:48.493 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:48.539 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:48.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:48.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:48.675 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:48.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:48.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:48.814 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:48.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:48.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:48.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:48.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:48.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 19 16 47362 47346 155.729 119.5 0.0116291 0.00641651 2024-09-21T10:05:49.058 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:49.097 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:49.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:49.199 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:49.240 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:49.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:49.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:49.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:49.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:49.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:49 smithi032 ceph-mon[105748]: pgmap v316: 89 pgs: 89 active+clean; 2.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-21T10:05:49.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:49.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:49 smithi032 ceph-mon[101299]: pgmap v316: 89 pgs: 89 active+clean; 2.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-21T10:05:49.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:49.483 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:49.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.521 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:49.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:49.569 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:05:49.569 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:48484 5.5 e118/112)", 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:49.565780+0000", 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.001054985, 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0010756990000000001, 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565780+0000", 2024-09-21T10:05:49.570 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565780+0000", 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565781+0000", 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.052e-06 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565816+0000", 2024-09-21T10:05:49.571 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5037999999999999e-05 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565833+0000", 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7564999999999999e-05 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565842+0000", 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4339999999999992e-06 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.572 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:48499 5.c 5.61901bbc (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:49.565911+0000", 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.00092412799999999997, 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00099366699999999994, 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 48499 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:05:49.573 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565911+0000", 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565911+0000", 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565912+0000", 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5060000000000001e-06 2024-09-21T10:05:49.574 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565936+0000", 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3867e-05 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565948+0000", 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1779e-05 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.565955+0000", 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2320000000000004e-06 2024-09-21T10:05:49.575 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:48483 5.5 e118/112 5:af56e798:::benchmark_data_smithi032_140417_object48482:head v 118'3038, mlcod=118'3036)", 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:49.563333+0000", 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0035016230000000001, 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003628947, 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:05:49.576 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.563333+0000", 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.563333+0000", 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.563516+0000", 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018287199999999999 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.577 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.563547+0000", 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0670999999999998e-05 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.563561+0000", 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4523e-05 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.563570+0000", 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0380000000000005e-06 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.578 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.563671+0000", 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000100524 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:49.563694+0000", 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3178999999999999e-05 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:05:49.579 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:05:49.580 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 3 2024-09-21T10:05:49.580 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:05:49.622 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:49.659 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:49.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:49.761 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:49.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:49 smithi045 ceph-mon[95976]: pgmap v316: 89 pgs: 89 active+clean; 2.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-21T10:05:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:49.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:49.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:49.897 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:49.935 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:49.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:05:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:05:49] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.51.0" 2024-09-21T10:05:49.961 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:05:49.960859+0000 min lat: 0.00110358 max lat: 0.172663 avg lat: 0.0064917 2024-09-21T10:05:49.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:05:49.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 20 16 49279 49263 153.933 119.812 0.00742717 0.0064917 2024-09-21T10:05:50.036 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:50.071 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:50.173 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:50.215 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:50.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:50.317 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:50.354 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:50.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:50.456 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:50.495 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:50.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:50.597 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:50.630 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:50.732 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:50.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.774 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:50.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:50.875 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:50.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:50.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 21 16 51205 51189 152.335 120.375 0.00970704 0.00655953 2024-09-21T10:05:51.016 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:51.053 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:51.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:51.134 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:51.155 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:51.169 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:05:51.186 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:51.288 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:51.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.330 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:51.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:51.431 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:51.465 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:51.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:51.567 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:51.602 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:51.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:51.704 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:51.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:51 smithi032 ceph-mon[105748]: pgmap v317: 89 pgs: 89 active+clean; 2.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-21T10:05:51.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:51 smithi032 ceph-mon[101299]: pgmap v317: 89 pgs: 89 active+clean; 2.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-21T10:05:51.733 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:51.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:51.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:51 smithi045 ceph-mon[95976]: pgmap v317: 89 pgs: 89 active+clean; 2.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-21T10:05:51.835 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:51.872 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:51.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 22 16 52767 52751 149.847 97.625 0.00719357 0.00666775 2024-09-21T10:05:51.974 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:52.011 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:52.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:52.113 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:52.142 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:52.244 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:52.280 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:52.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:52.381 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:52.417 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:52.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:52.519 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:52.553 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:52.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:52.655 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:52.691 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:52.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:52.793 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:52.838 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:52.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:52.940 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:52.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 23 16 54508 54492 148.063 108.812 0.0129042 0.00674941 2024-09-21T10:05:52.982 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:52.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:53.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:53.120 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:53.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:53.222 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:53.244 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:53.346 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:53.380 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:53.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:53.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:53.520 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:53.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:53.622 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:53.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:53.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:53 smithi032 ceph-mon[101299]: pgmap v318: 89 pgs: 89 active+clean; 2.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-21T10:05:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:53 smithi032 ceph-mon[105748]: pgmap v318: 89 pgs: 89 active+clean; 2.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-21T10:05:53.763 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:53.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:53 smithi045 ceph-mon[95976]: pgmap v318: 89 pgs: 89 active+clean; 2.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-21T10:05:53.807 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:53.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:53.909 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:53.945 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:53.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:53.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 24 16 56044 56028 145.893 96 0.007428 0.00684891 2024-09-21T10:05:54.046 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:54.073 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:54.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:54.174 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:54.212 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:54.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:54.314 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:54.349 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:54.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:54.451 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:54.492 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:54.594 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:54.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:54.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:54.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:54.877 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.910 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:54.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:54.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:05:54.911 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:54.959 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:05:54.959 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:54.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 25 16 57612 57596 143.977 98 0.00926367 0.00694058 2024-09-21T10:05:54.997 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:05:54.998 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:55.011 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:55.038 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:05:55.038 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:55.046 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:55.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:55.078 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:05:55.078 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:55.116 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:05:55.117 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:55.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:55.154 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:05:55.154 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:55.177 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:55.191 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:05:55.191 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:55.279 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:55.318 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:55.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:55.420 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:55.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:55.457 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:05:55.558 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:55.594 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:55.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:55.696 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:55.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:55 smithi032 ceph-mon[101299]: pgmap v319: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-21T10:05:55.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2436179976' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:55.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3208500980' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:55.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:55 smithi032 ceph-mon[105748]: pgmap v319: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-21T10:05:55.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2436179976' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:55.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3208500980' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:55.735 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:55.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:55.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:55 smithi045 ceph-mon[95976]: pgmap v319: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-21T10:05:55.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2436179976' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:55.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3208500980' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:55.836 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:55.875 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:55.962 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 26 16 59326 59310 142.559 107.125 0.00990562 0.00701004 2024-09-21T10:05:55.978 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:56.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.016 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:56.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:56.117 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:56.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.157 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:56.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:56.258 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:56.291 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:56.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:56.393 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:56.435 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:56.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:56.537 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:56.575 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:56.650 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:56 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:05:56.651 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:56 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:05:56.651 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3208500980' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:56.651 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:56 smithi032 ceph-mon[101299]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T10:05:56.651 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:56 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:05:56.651 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:56 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:05:56.651 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3208500980' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:56.651 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:56 smithi032 ceph-mon[105748]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T10:05:56.677 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:56.712 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:56.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:56.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:56 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:05:56.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:56 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:05:56.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3208500980' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:56.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:56 smithi045 ceph-mon[95976]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T10:05:56.814 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:56.852 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:56.954 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:56.964 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 27 16 61087 61071 141.346 110.062 0.016088 0.0070698 2024-09-21T10:05:57.000 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:57.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:57.101 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:57.141 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:57.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:57.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:57.278 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:57.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:57.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:57.416 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:57.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:57.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:57.553 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:57 smithi032 ceph-mon[101299]: pgmap v321: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-21T10:05:57.553 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:57 smithi032 ceph-mon[105748]: pgmap v321: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-21T10:05:57.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.554 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:57.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:57.655 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:57.698 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:57.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:57.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:57 smithi045 ceph-mon[95976]: pgmap v321: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-21T10:05:57.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:57.828 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:57.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:57.930 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:57.964 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 28 16 62666 62650 139.823 98.6875 0.00591695 0.00714697 2024-09-21T10:05:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:57.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:58.070 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:58.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.102 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:58.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:58.203 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:58.242 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:58.311 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:58.344 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:58.381 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:58.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:58.473 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:05:58.483 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:58.491 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:58 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/436560091' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:05:58.491 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:58 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/436560091' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:05:58.528 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:58.630 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:58.664 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:58.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:58.766 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:58.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:58 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/436560091' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:05:58.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:58.902 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:58.941 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:58.964 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 29 16 64322 64306 138.57 103.5 0.0067665 0.0072122 2024-09-21T10:05:59.017 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:05:59.044 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:59.076 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:59.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:59.178 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:59.216 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:59.317 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:59.362 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:59.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:59.464 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:59.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:59.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:59.643 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:59.744 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:59.745 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:59 smithi032 ceph-mon[101299]: pgmap v322: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-09-21T10:05:59.745 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/436560091' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:05:59.745 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:05:59 smithi032 ceph-mon[101299]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T10:05:59.746 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:05:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:05:59] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.51.0" 2024-09-21T10:05:59.746 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:59 smithi032 ceph-mon[105748]: pgmap v322: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-09-21T10:05:59.746 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/436560091' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:05:59.746 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:05:59 smithi032 ceph-mon[105748]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T10:05:59.779 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:05:59.784 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:05:59.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:59 smithi045 ceph-mon[95976]: pgmap v322: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-09-21T10:05:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/436560091' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:05:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:05:59 smithi045 ceph-mon[95976]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T10:05:59.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:59.839 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:59.886 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:59.933 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:05:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:59.965 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 30 16 65893 65877 137.223 98.1875 0.00771303 0.00728238 2024-09-21T10:06:00.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:00.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:00.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:00.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:00.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:00.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:00.281 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:06:00.324 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:00.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:00.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:00.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:00.485 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:00.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:00.587 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:00.631 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:00.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:00.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:00.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:00.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:00.925 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:00.965 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 31 16 67243 67227 135.518 84.375 0.00376218 0.00737431 2024-09-21T10:06:01.027 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:01.078 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:01.180 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:01.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:01.301 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:06:01.301 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:06:01.312 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:01.336 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:01.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:01.438 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:01.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:01.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:01.579 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:01.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.615 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:01.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:01.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:01 smithi032 ceph-mon[101299]: pgmap v324: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-21T10:06:01.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:01 smithi032 ceph-mon[105748]: pgmap v324: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-21T10:06:01.715 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:01.752 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:01.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:01 smithi045 ceph-mon[95976]: pgmap v324: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-21T10:06:01.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:01.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:01.965 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 32 16 68658 68642 134.046 88.4375 0.0105949 0.00745561 2024-09-21T10:06:01.991 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.034 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:02.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:02.135 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:02.180 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:02.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:02.282 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:02.323 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:02.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:02.424 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:02.463 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:02.565 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:02.601 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:02.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:02.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:02.745 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:02.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:02.793 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:06:02.811 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:02.847 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:02.884 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:02.965 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 33 16 70076 70060 132.668 88.625 0.00655372 0.00753214 2024-09-21T10:06:02.986 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.024 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:03.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:03.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:03.167 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:03.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:03.269 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:03.314 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:03.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:03.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:03.445 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:03.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:03.547 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:03.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:03.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:03.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:03.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:03.755 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[105748]: pgmap v325: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:06:03.755 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:06:03.755 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:03.755 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:03.756 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:03.756 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:03.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[101299]: pgmap v325: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:06:03.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:06:03.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:03.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:03.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:03.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:03 smithi045 ceph-mon[95976]: pgmap v325: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:06:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:06:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:03.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:03.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:03.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:03.890 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:03.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:03.965 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 34 16 71475 71459 131.338 87.4375 0.0140605 0.00760837 2024-09-21T10:06:03.992 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:04.027 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:04.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:04.129 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:04.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.164 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:04.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:04.265 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:04.299 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:04.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:04.401 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:04.436 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:04.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:04.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:04.539 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:06:04.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.576 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:04.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:04.677 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.716 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:04.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:04.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:04 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3846756042' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:04.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:04 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:04.803 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:04 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3846756042' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:04.803 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:04 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:04.803 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:04 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3846756042' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:04.803 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:04 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:04.817 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:04.858 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:04.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:04.960 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:04.965 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 35 16 72888 72872 130.108 88.3125 0.00750338 0.00768101 2024-09-21T10:06:05.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.011 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:05.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:05.112 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:05.151 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:05.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:05.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:05.295 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:05.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:05.396 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:05.434 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:05.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:05.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:05 smithi045 ceph-mon[95976]: pgmap v326: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-21T10:06:05.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:05 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:05.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:05 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:06:05.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:05 smithi045 ceph-mon[95976]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T10:06:05.536 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:05.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:05.676 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:05 smithi032 ceph-mon[105748]: pgmap v326: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-21T10:06:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:05 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:05 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:06:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:05 smithi032 ceph-mon[105748]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T10:06:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:05 smithi032 ceph-mon[101299]: pgmap v326: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-21T10:06:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:05 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:05 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:06:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:05 smithi032 ceph-mon[101299]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T10:06:05.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.717 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:05.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:05.751 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:05.817 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:05.858 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:05.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:05.960 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:05.965 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 36 16 74303 74287 128.951 88.4375 0.0202101 0.00774967 2024-09-21T10:06:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.005 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:06.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:06.106 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:06.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.151 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:06.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:06.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:06.302 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:06.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:06.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:06.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:06.547 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:06.587 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:06.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:06.689 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:06.723 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:06.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:06.825 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.864 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:06.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:06.964 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:06.966 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 37 16 75689 75673 127.807 86.625 0.00662819 0.00781899 2024-09-21T10:06:07.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:07.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:07.115 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:07.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:07.262 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:07.292 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:07.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:07.329 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:06:07.393 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.446 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:07.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:07.547 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:07.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:07.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:07.684 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:07.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:07 smithi032 ceph-mon[101299]: pgmap v328: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:06:07.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:07 smithi032 ceph-mon[105748]: pgmap v328: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:06:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.730 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:07.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:07 smithi045 ceph-mon[95976]: pgmap v328: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:06:07.831 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:07.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.870 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:07.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:07.916 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:07.966 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 38 16 76966 76950 126.543 79.8125 0.01563 0.00789542 2024-09-21T10:06:07.971 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:08.009 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:08.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:08.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:08.150 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:08.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:08.252 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.289 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:08.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:08.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.442 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:08.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:08.509 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:06:08.544 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:08.575 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:08.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:08.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:08.700 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:08 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/149925205' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:06:08.700 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:08 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:06:08.701 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:08 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/149925205' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:06:08.701 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:08 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:06:08.724 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:08.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:08.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:08 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/149925205' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:06:08.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:08 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:06:08.825 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:08.866 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:08.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:08.966 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 39 16 78361 78345 125.534 87.1875 0.00462396 0.00796103 2024-09-21T10:06:08.969 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:08.999 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:09.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:09.101 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:09.122 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:09.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:09.224 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:09.262 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:09.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:09.363 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:09.402 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:09.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:09.504 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:09.541 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:09.643 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:09.683 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:09.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:09 smithi045 ceph-mon[95976]: pgmap v329: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:06:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:09 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:06:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:09 smithi045 ceph-mon[95976]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T10:06:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:09.785 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:09.818 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:09.920 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:09.954 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:06:09] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T10:06:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[105748]: pgmap v329: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:06:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:06:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[105748]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T10:06:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[101299]: pgmap v329: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:06:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:06:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[101299]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T10:06:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:09.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:09.966 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:06:09.964977+0000 min lat: 0.00110358 max lat: 0.172663 avg lat: 0.00803011 2024-09-21T10:06:09.966 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:06:09.966 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 40 16 79672 79656 124.444 81.9375 0.0187148 0.00803011 2024-09-21T10:06:10.055 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:10.091 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:10.192 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:10.228 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:10.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:10.279 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:06:10.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:10.367 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:10.468 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:10.521 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:10.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:10.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:10.662 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:10.763 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:10.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.807 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:10.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:10.889 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:06:10.889 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:10.908 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:10.924 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:06:10.924 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:10.946 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:10.959 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:06:10.959 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:10.966 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 41 16 80988 80972 123.415 82.25 0.0135589 0.0080977 2024-09-21T10:06:10.994 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:06:10.994 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:11.028 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:06:11.028 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:06:11.048 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:11.068 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:06:11.069 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:11.082 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:11.103 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:06:11.103 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:11.140 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:06:11.140 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:11.184 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:11.223 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:11.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:11.325 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:11.359 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:11.460 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:11.501 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:11.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:11.603 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:11.647 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:11.749 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:11.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:11 smithi045 ceph-mon[95976]: pgmap v331: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:06:11.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:11 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3782544081' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:11.790 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:11.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:11.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:11.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:11 smithi032 ceph-mon[101299]: pgmap v331: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:06:11.913 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:11 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3782544081' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:11.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:11 smithi032 ceph-mon[105748]: pgmap v331: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:06:11.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:11 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3782544081' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:11.926 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:11.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:11.966 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 42 16 82360 82344 122.518 85.75 0.00489893 0.00815755 2024-09-21T10:06:12.028 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.063 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:12.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:12.164 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:12.204 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:12.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:12.268 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:12.275 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:12.307 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:12.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:12.463 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:12.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:12.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:12.599 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:12.633 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:12.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:12.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:12.781 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:12.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:12.883 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:12.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:12.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:12.966 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 43 16 83701 83685 121.618 83.8125 0.00777231 0.00821761 2024-09-21T10:06:13.022 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:13.053 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:13.155 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:13.191 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:13.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:13.293 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:13.329 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:13.432 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:13.468 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:13.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:13.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:13.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:13.676 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:13 smithi032 ceph-mon[101299]: pgmap v332: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:13.676 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:13 smithi032 ceph-mon[105748]: pgmap v332: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:13.704 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:13.747 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:13.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:13 smithi045 ceph-mon[95976]: pgmap v332: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:13.849 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:13.886 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:13.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:13.968 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 44 16 85001 84985 120.697 81.25 0.0152356 0.00828019 2024-09-21T10:06:13.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:14.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:14.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:14.132 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:14.185 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:14.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:14.287 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:14.331 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:14.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:14.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:14.466 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:14.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:14.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:14.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.615 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:14.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:14.716 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:14.756 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:06:14.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:14.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:14.879 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:14.920 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:14.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:14.967 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 45 16 86203 86187 119.684 75.125 0.00662679 0.00835019 2024-09-21T10:06:15.022 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:15.083 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:15.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:15.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:15.221 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:15.322 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:15.358 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:15.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:15.460 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:15.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:15.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:15.589 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:15.623 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:15.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:15.725 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:15.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.768 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:15.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:15.784 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:15 smithi032 ceph-mon[105748]: pgmap v333: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T10:06:15.785 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:15 smithi032 ceph-mon[101299]: pgmap v333: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T10:06:15.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:15 smithi045 ceph-mon[95976]: pgmap v333: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T10:06:15.870 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:15.893 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:15.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:15.967 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 46 16 87238 87222 118.488 64.6875 0.00723344 0.00841794 2024-09-21T10:06:15.995 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:16.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:16.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:16.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:16.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:16.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:16.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:16.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:16.412 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:16.449 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:16.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:16.552 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:16.591 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:16.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:16.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:16.710 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:06:16.710 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:06:16.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:06:16.710 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:06:16.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.710 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:26401 5.c 5:33d0e714:::benchmark_data_smithi032_140417_object26400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:16.711 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:38.419587+0000", 2024-09-21T10:06:16.711 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.353285835999998, 2024-09-21T10:06:16.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.059839874000000001, 2024-09-21T10:06:16.711 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.711 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.711 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.711 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.711 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.711 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.712 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26401 2024-09-21T10:06:16.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.712 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.712 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.712 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.419587+0000", 2024-09-21T10:06:16.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.419587+0000", 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.419588+0000", 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.426e-06 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.419611+0000", 2024-09-21T10:06:16.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3059000000000001e-05 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.419623+0000", 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2069000000000001e-05 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.419631+0000", 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4580000000000004e-06 2024-09-21T10:06:16.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.419684+0000", 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3182e-05 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.419829+0000", 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000145675 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.419957+0000", 2024-09-21T10:06:16.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012802500000000001 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.420995+0000", 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001037699 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.422870+0000", 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001874403 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.716 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.479126+0000", 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.056256144000000001 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.479377+0000", 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00025145499999999998 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.479426+0000", 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9279e-05 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:26403 5.c 5:3288a15f:::benchmark_data_smithi032_140417_object26402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:38.421415+0000", 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.351457431, 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.059141103, 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26403 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.421415+0000", 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.421415+0000", 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.421416+0000", 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.562e-06 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.421439+0000", 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2612e-05 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.421497+0000", 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7787000000000001e-05 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.421506+0000", 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7690000000000002e-06 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.421584+0000", 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8017999999999997e-05 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.421720+0000", 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013590600000000001 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.421841+0000", 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000121908 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.423324+0000", 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014822220000000001 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.423566+0000", 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024276899999999999 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.480451+0000", 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.056884479000000002 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.480508+0000", 2024-09-21T10:06:16.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7487e-05 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.480556+0000", 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7583999999999997e-05 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:26456 5.c 5:33b5f9a0:::benchmark_data_smithi032_140417_object26455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:38.498958+0000", 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.273914683999998, 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.163161948, 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.723 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26456 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.498958+0000", 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.724 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.498958+0000", 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.498959+0000", 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2249999999999999e-06 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.498978+0000", 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8833000000000001e-05 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.725 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.498989+0000", 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1654999999999999e-05 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.498995+0000", 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6810000000000003e-06 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.499049+0000", 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3857000000000003e-05 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.499160+0000", 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000110952 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.499256+0000", 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6583999999999995e-05 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.500059+0000", 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00080274000000000003 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.500356+0000", 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029635399999999998 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.662020+0000", 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16166404300000001 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.662076+0000", 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6307000000000001e-05 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.728 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.662120+0000", 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3717000000000002e-05 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:34935 5.8 e117/112 5:173e5851:::benchmark_data_smithi032_140417_object34934:head v 117'2197, mlcod=117'2194)", 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:42.478732+0000", 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 33.294140605999999, 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.069823891999999999, 2024-09-21T10:06:16.729 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.478732+0000", 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.478732+0000", 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.478733+0000", 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1200000000000001e-06 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.478779+0000", 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6279000000000002e-05 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.478791+0000", 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2138e-05 2024-09-21T10:06:16.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.478799+0000", 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9359999999999999e-06 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.478854+0000", 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4549999999999998e-05 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.732 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.478873+0000", 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8853999999999999e-05 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.548488+0000", 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.069615417999999998 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.548556+0000", 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7596999999999997e-05 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.733 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.734 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.734 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.734 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:34930 5.0 e117/112 5:0b1d4d0a:::benchmark_data_smithi032_140417_object34929:head v 117'2185, mlcod=117'2182)", 2024-09-21T10:06:16.734 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:42.479090+0000", 2024-09-21T10:06:16.734 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 33.293782372000003, 2024-09-21T10:06:16.734 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.069510169999999996, 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.479090+0000", 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.479090+0000", 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.479091+0000", 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9100000000000003e-07 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.479118+0000", 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6860999999999999e-05 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.545447+0000", 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.066329733000000002 2024-09-21T10:06:16.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.545456+0000", 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3590000000000008e-06 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.545511+0000", 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5251999999999999e-05 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.545531+0000", 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0142000000000001e-05 2024-09-21T10:06:16.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.548560+0000", 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0030285479999999998 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.548600+0000", 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0484000000000001e-05 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:34950 5.7 e117/112 5:ee5b2d68:::benchmark_data_smithi032_140417_object34949:head v 117'2181, mlcod=117'2179)", 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:42.484524+0000", 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 33.288347934000001, 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.062934087999999999, 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.484524+0000", 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.739 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.484524+0000", 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.484526+0000", 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.874e-06 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.484567+0000", 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0212000000000001e-05 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.740 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.545431+0000", 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.060864835999999999 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.545445+0000", 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3577999999999999e-05 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.545505+0000", 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9985000000000003e-05 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.545525+0000", 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9789999999999999e-05 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.547398+0000", 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001873008 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.547459+0000", 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0804999999999998e-05 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.742 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:35267 5.c 5:3c938db8:::benchmark_data_smithi032_140417_object35266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:42.666017+0000", 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 33.106854929000001, 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.037675437999999999, 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 35267 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.743 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.666017+0000", 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.666017+0000", 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.744 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.666019+0000", 2024-09-21T10:06:16.745 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4050000000000001e-06 2024-09-21T10:06:16.745 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.745 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.745 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.745 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.666041+0000", 2024-09-21T10:06:16.745 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2084999999999999e-05 2024-09-21T10:06:16.745 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.745 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.745 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.666052+0000", 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1165999999999999e-05 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.666059+0000", 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8299999999999998e-06 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.666112+0000", 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2803999999999998e-05 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.746 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.666200+0000", 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8418000000000006e-05 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.666278+0000", 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7484000000000006e-05 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.702727+0000", 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.036448979999999999 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.747 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.702810+0000", 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3771999999999997e-05 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.703540+0000", 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00072967500000000005 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.703628+0000", 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7608999999999994e-05 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.748 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.703693+0000", 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5209999999999994e-05 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:36179 5.c 5:32264446:::benchmark_data_smithi032_140417_object36178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:43.163869+0000", 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 32.609003432000002, 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.052375007000000001, 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.749 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 36179 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.163869+0000", 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.750 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.163869+0000", 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.163870+0000", 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.279e-06 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.751 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.163891+0000", 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1282000000000001e-05 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.163903+0000", 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1007e-05 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.163909+0000", 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7730000000000002e-06 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.752 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.164027+0000", 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000117434 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.164153+0000", 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000126191 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.164305+0000", 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015227400000000001 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.166410+0000", 2024-09-21T10:06:16.753 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0021046900000000002 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.167000+0000", 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00059040500000000001 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.216032+0000", 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.049031883999999998 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.754 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.216151+0000", 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0001189 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.216244+0000", 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2887999999999996e-05 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:36180 5.c 5:3d7167b5:::benchmark_data_smithi032_140417_object36179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:43.164056+0000", 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 32.608816453000003, 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0535968, 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.755 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 36180 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.164056+0000", 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.164056+0000", 2024-09-21T10:06:16.756 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.164057+0000", 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8400000000000002e-07 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.164080+0000", 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2849000000000001e-05 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.164090+0000", 2024-09-21T10:06:16.757 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0625000000000001e-05 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.164097+0000", 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7109999999999998e-06 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.165498+0000", 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001401357 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.758 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.165610+0000", 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000111154 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.165741+0000", 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013094199999999999 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.166553+0000", 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00081288499999999995 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.759 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.167121+0000", 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00056722900000000004 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.217596+0000", 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.050475346999999997 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.217628+0000", 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1608e-05 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.217653+0000", 2024-09-21T10:06:16.760 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5109e-05 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:37737 5.c 5:31977b1b:::benchmark_data_smithi032_140417_object37736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:43.959610+0000", 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.813262810000001, 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.061022431000000002, 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.761 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 37737 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.959610+0000", 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.959610+0000", 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.762 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.959611+0000", 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.573e-06 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.959635+0000", 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4355e-05 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.959646+0000", 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0180000000000001e-05 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.959653+0000", 2024-09-21T10:06:16.763 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4950000000000002e-06 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.959705+0000", 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2235999999999999e-05 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.959830+0000", 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000124616 2024-09-21T10:06:16.764 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.959906+0000", 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5996999999999998e-05 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.016867+0000", 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.056961073000000001 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018369+0000", 2024-09-21T10:06:16.765 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0015020929999999999 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.020533+0000", 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002163693 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.020591+0000", 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8406999999999998e-05 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.020632+0000", 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0713000000000001e-05 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:37738 5.0 e117/112 5:04f0838a:::benchmark_data_smithi032_140417_object37737:head v 117'2377, mlcod=117'2375)", 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:43.960764+0000", 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.812108295000002, 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.057670335000000003, 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.767 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960764+0000", 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960764+0000", 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960765+0000", 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.094e-06 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.768 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960785+0000", 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9788000000000001e-05 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960793+0000", 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2309999999999996e-06 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960798+0000", 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5190000000000003e-06 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.769 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960832+0000", 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4215999999999999e-05 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960849+0000", 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6722999999999999e-05 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018383+0000", 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.057534455999999998 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.770 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018434+0000", 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1307999999999997e-05 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44543.0:37737 5.c e117/112 ondisk, result = 0)", 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:43.962766+0000", 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.810106387000001, 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.055610357999999999, 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.771 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.962766+0000", 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.962766+0000", 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.962767+0000", 2024-09-21T10:06:16.772 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9500000000000002e-07 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.962770+0000", 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.348e-06 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.016295+0000", 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.053525387000000001 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.016307+0000", 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1433e-05 2024-09-21T10:06:16.773 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018355+0000", 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0020477859999999998 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018363+0000", 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3590000000000008e-06 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018376+0000", 2024-09-21T10:06:16.774 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.345e-05 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:37742 5.7 e117/112 5:e82fdb17:::benchmark_data_smithi032_140417_object37741:head v 117'2372, mlcod=117'2370)", 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:43.963847+0000", 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.809025409, 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.054753260999999998, 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.775 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.963847+0000", 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.963847+0000", 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.963848+0000", 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6599999999999996e-07 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.776 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.963915+0000", 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7724000000000004e-05 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.015083+0000", 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.051167762999999998 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.015098+0000", 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4977000000000001e-05 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.777 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.015202+0000", 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000103674 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.015230+0000", 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.8402999999999999e-05 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018528+0000", 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003297725 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.778 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018600+0000", 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2329000000000005e-05 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:39800 5.c 5:33417200:::benchmark_data_smithi032_140417_object39799:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:45.095895+0000", 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 30.676977798999999, 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.070148110999999999, 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.779 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 39800 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.095895+0000", 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.780 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.095895+0000", 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.095896+0000", 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4079999999999999e-06 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.095922+0000", 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6508000000000001e-05 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.781 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.095933+0000", 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0504e-05 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.095939+0000", 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4080000000000003e-06 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.095998+0000", 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8285000000000002e-05 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.782 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.096124+0000", 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012633800000000001 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.096254+0000", 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000130431 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.097693+0000", 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014388370000000001 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.783 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.097787+0000", 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3400999999999996e-05 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.165947+0000", 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.068160160999999997 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.784 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.166001+0000", 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.448e-05 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.166043+0000", 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1350000000000002e-05 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:40102 5.c 5:3e149406:::benchmark_data_smithi032_140417_object40101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:45.327170+0000", 2024-09-21T10:06:16.785 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 30.445702122, 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.059970628999999998, 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 40102 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.786 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.327170+0000", 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.327170+0000", 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.327172+0000", 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3370000000000001e-06 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.327190+0000", 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8879999999999999e-05 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.327200+0000", 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0499999999999997e-06 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.327205+0000", 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7180000000000001e-06 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.788 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.327303+0000", 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7560999999999996e-05 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.327472+0000", 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016941300000000001 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.327637+0000", 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000164668 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.329098+0000", 2024-09-21T10:06:16.789 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014614109999999999 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.329393+0000", 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029459799999999997 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387022+0000", 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.057629095999999998 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.790 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387090+0000", 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8380000000000006e-05 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387141+0000", 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0516999999999998e-05 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:40111 5.7 e117/112 5:e9b47b06:::benchmark_data_smithi032_140417_object40110:head v 117'2544, mlcod=117'2541)", 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:45.331565+0000", 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 30.441307747, 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.055647146000000001, 2024-09-21T10:06:16.791 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.331565+0000", 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.331565+0000", 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.792 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.331565+0000", 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5000000000000003e-07 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.331621+0000", 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5714999999999998e-05 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.331628+0000", 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0010000000000004e-06 2024-09-21T10:06:16.793 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.331632+0000", 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4769999999999997e-06 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.331673+0000", 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0794999999999998e-05 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.331705+0000", 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1457000000000001e-05 2024-09-21T10:06:16.794 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332642+0000", 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00093755599999999998 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387212+0000", 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.054569594999999999 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.795 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44543.0:40102 5.c e117/112 ondisk, result = 0)", 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:45.332558+0000", 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 30.440314270000002, 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.054592066000000002, 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332558+0000", 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332558+0000", 2024-09-21T10:06:16.796 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332559+0000", 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7400000000000002e-07 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332578+0000", 2024-09-21T10:06:16.797 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9239999999999999e-05 2024-09-21T10:06:16.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.798 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.798 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.798 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332589+0000", 2024-09-21T10:06:16.798 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.077e-05 2024-09-21T10:06:16.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332595+0000", 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6000000000000003e-06 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332648+0000", 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2602999999999998e-05 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.799 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387004+0000", 2024-09-21T10:06:16.800 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.054355630000000002 2024-09-21T10:06:16.800 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.800 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.800 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.800 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387150+0000", 2024-09-21T10:06:16.800 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014644899999999999 2024-09-21T10:06:16.800 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.800 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.800 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:40113 5.0 e117/112 5:065f77a4:::benchmark_data_smithi032_140417_object40112:head v 117'2526, mlcod=117'2524)", 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:45.332987+0000", 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 30.439885128, 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.058685253, 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.801 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332987+0000", 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332987+0000", 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332988+0000", 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9299999999999997e-07 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.802 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.333019+0000", 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1466e-05 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387266+0000", 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.054246573999999999 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387279+0000", 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3039e-05 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.803 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387328+0000", 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9147000000000002e-05 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387352+0000", 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4111e-05 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.391619+0000", 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0042667080000000001 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.804 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.391672+0000", 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3415000000000003e-05 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:52108 5.c 5:393de88f:::benchmark_data_smithi032_140417_object52107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:51.456327+0000", 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 24.316545753, 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.113532094, 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.805 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 52108 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.456327+0000", 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.806 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.456327+0000", 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.456328+0000", 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.621e-06 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.456355+0000", 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6358000000000001e-05 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.807 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.456371+0000", 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6059e-05 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.456380+0000", 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3999999999999998e-06 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.456434+0000", 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4361999999999999e-05 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.808 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.456518+0000", 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4005999999999995e-05 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.456597+0000", 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.894e-05 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.457869+0000", 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0012720889999999999 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.809 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.459214+0000", 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001344593 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.569725+0000", 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.110510715 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.569810+0000", 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5068000000000006e-05 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.810 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.569859+0000", 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8882999999999999e-05 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:52409 5.c 5:3cb6c61b:::benchmark_data_smithi032_140417_object52408:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:51.718364+0000", 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 24.054508500000001, 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.057167733999999998, 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:16.811 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 52409 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.718364+0000", 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.718364+0000", 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.812 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.718365+0000", 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.22e-06 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.718382+0000", 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7269e-05 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.718390+0000", 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7950000000000008e-06 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.718395+0000", 2024-09-21T10:06:16.813 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.826e-06 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.718436+0000", 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0828999999999998e-05 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.718562+0000", 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000126546 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.718696+0000", 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013317299999999999 2024-09-21T10:06:16.814 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.720214+0000", 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001518939 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.721146+0000", 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00093117099999999995 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.775390+0000", 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.054244440999999997 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.815 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.775484+0000", 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3610999999999995e-05 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:51.775532+0000", 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7914000000000002e-05 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:16.816 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:06:16.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.817 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:16.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:16.918 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:16.958 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:16.967 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 47 16 88503 88487 117.649 79.0625 0.0174806 0.00849419 2024-09-21T10:06:17.060 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:17.082 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:17.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:17.183 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:17.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:17.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:17.321 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:17.361 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:17.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:17.464 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.499 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:17.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:17.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:17.628 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:17.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:17.730 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:17.766 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:17.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:17.868 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:17.907 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:17 smithi032 ceph-mon[105748]: pgmap v334: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-09-21T10:06:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:17 smithi032 ceph-mon[101299]: pgmap v334: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-09-21T10:06:17.967 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 48 16 89668 89652 116.715 72.8125 0.0104548 0.00856292 2024-09-21T10:06:18.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:18.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:17 smithi045 ceph-mon[95976]: pgmap v334: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-09-21T10:06:18.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.048 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:18.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:18.149 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:18.179 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:18.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:18.281 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:18.317 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:18.419 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:18.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.441 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:18.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:18.542 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:18.584 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:18.686 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:18.722 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:18.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:18.824 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:18.862 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:18.964 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:18.968 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 49 16 90658 90642 115.596 61.875 0.00961037 0.00864579 2024-09-21T10:06:18.999 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:19.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:19.101 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:19.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.137 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:19.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:19.238 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:19.277 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:19.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:19.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:19.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:19.516 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:19.555 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:19.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:19.656 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:19.695 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:19 smithi045 ceph-mon[95976]: pgmap v335: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-21T10:06:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:19.797 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:19.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:19.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:19.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:19 smithi032 ceph-mon[101299]: pgmap v335: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-21T10:06:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:19.959 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:06:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:06:19] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T10:06:19.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:19 smithi032 ceph-mon[105748]: pgmap v335: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-21T10:06:19.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:19.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:19.970 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 50 16 91773 91757 114.673 69.6875 0.0184631 0.00871549 2024-09-21T10:06:19.998 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:20.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:20.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.135 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:20.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:20.235 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:20.270 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:20.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:20.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:20.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:20.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:20.513 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:20.547 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:20.649 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:20.689 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:20.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:20.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:20.828 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:20.930 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:20.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.964 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:20.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:20.970 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 51 16 93026 93010 113.96 78.3125 0.0158722 0.00877026 2024-09-21T10:06:21.064 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:21.102 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:21.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:21.204 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:21.244 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:21.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.383 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:21.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:21.484 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:21.523 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:21.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:21.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:21.663 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:21 smithi032 ceph-mon[105748]: pgmap v336: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-21T10:06:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:21 smithi032 ceph-mon[101299]: pgmap v336: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-21T10:06:21.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:21.781 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:06:21.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:21 smithi045 ceph-mon[95976]: pgmap v336: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-21T10:06:21.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:21.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:21.904 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:21.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:21.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:21.970 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 52 16 94274 94258 113.268 78 0.00841866 0.00882337 2024-09-21T10:06:22.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:22.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.078 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:22.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:22.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.219 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:22.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:22.320 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:22.352 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:22.454 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:22.492 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:22.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:22.593 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:22.627 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:22.728 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:22.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:22.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:22.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:22.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:22.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:22.970 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 53 16 95479 95463 112.552 75.3125 0.00904534 0.00887973 2024-09-21T10:06:22.998 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:23.038 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:23.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:23.140 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:23.178 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:23.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:23.279 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:23.287 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:23.341 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:23.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:23.443 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:23.482 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:23.495 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:06:23.584 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:23.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:23.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:23.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:23 smithi032 ceph-mon[105748]: pgmap v337: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-21T10:06:23.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/828601674' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:23.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:23 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:23.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:23 smithi032 ceph-mon[101299]: pgmap v337: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-21T10:06:23.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/828601674' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:23.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:23 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:23.726 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:23.772 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:23.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:23 smithi045 ceph-mon[95976]: pgmap v337: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-21T10:06:23.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/828601674' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:23.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:23 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:23.874 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:23.914 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:23.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:23.970 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 54 16 96631 96615 111.801 72 0.00857062 0.00893929 2024-09-21T10:06:24.016 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:24.050 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:24.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:24.152 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:24.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:24.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:24.296 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:24.335 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:24.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:24.437 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:24.471 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:24.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:24.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:24.613 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:24.715 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:24.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:24.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:24 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:24 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:24 smithi045 ceph-mon[95976]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T10:06:24.838 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:24 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:24.838 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:24 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:24.838 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:24 smithi032 ceph-mon[101299]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T10:06:24.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:24 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:24.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:24 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:24.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:24 smithi032 ceph-mon[105748]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T10:06:24.847 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:24.878 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:24.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:24.970 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 55 16 97834 97818 111.135 75.1875 0.0112526 0.00899288 2024-09-21T10:06:24.979 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:25.019 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:25.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:25.121 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:25.150 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:25.251 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:25.292 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:25.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:25.393 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:25.426 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:25.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:25.493 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:06:25.528 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.558 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:25.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:25.659 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:25.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.702 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:25.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:25.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:25 smithi045 ceph-mon[95976]: pgmap v339: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-21T10:06:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1525173190' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:06:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:25 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:06:25.803 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:25.825 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:25.927 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:25.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:25 smithi032 ceph-mon[105748]: pgmap v339: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-21T10:06:25.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1525173190' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:06:25.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:25 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:06:25.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:25 smithi032 ceph-mon[101299]: pgmap v339: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-21T10:06:25.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1525173190' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:06:25.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:25 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:06:25.964 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:25.970 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 56 16 98823 98807 110.255 61.8125 0.0159708 0.00906515 2024-09-21T10:06:26.066 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:26.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:26.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:26.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:26.240 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:26.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:26.342 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:26.364 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:26.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:26.465 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:26.500 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:26.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:26.602 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:26.637 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:26.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:26.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:26.760 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:26.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:26.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:26 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:06:26.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:26 smithi045 ceph-mon[95976]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T10:06:26.862 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:26.883 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:06:26.907 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:26.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:26.919 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:26 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:06:26.919 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:26 smithi032 ceph-mon[101299]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T10:06:26.919 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:26 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:06:26.919 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:26 smithi032 ceph-mon[105748]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T10:06:26.970 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 57 16 99816 99800 109.409 62.0625 0.0116014 0.00913481 2024-09-21T10:06:27.009 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:27.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:27.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:27.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:27.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.182 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:27.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:27.283 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:27.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:27.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:27.406 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:27.444 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:27.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:27.546 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:27.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.586 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:27.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:27.686 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:27.719 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:27.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:27.778 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:27 smithi032 ceph-mon[105748]: pgmap v341: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-21T10:06:27.778 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:27 smithi032 ceph-mon[101299]: pgmap v341: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-21T10:06:27.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:27 smithi045 ceph-mon[95976]: pgmap v341: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-21T10:06:27.822 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:27.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.865 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:27.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:27.966 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:27.970 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 58 16 100901 100885 108.692 67.8125 0.01232 0.00919508 2024-09-21T10:06:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.006 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:28.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:28.107 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:28.125 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:28.153 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:28.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:28.255 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:28.302 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:28.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:28.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:28.451 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:28.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:28.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:28.591 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:28.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:28.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:06:28.693 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:28.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.740 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:28.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:28.841 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:28.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:28.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:28.971 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 59 16 101975 101959 107.987 67.125 0.0198165 0.00925542 2024-09-21T10:06:28.995 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:29.029 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:29.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:29.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:29.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:29.263 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:29.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:29.306 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[105748]: pgmap v342: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:06:29.306 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:29.306 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:29.306 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:29.306 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:29.306 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:29.306 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[101299]: pgmap v342: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:06:29.306 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:29.306 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:29.307 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:29.307 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:29.307 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:29.406 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:29.460 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:29 smithi045 ceph-mon[95976]: pgmap v342: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:06:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:29 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:29 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:29 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:29 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:29 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:06:29.558 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:06:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:06:29] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T10:06:29.563 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:29.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.610 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:29.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:29.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:29.745 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:29.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:29.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:29.880 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:29.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:29.971 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:06:29.970100+0000 min lat: 0.00110358 max lat: 0.182943 avg lat: 0.00931215 2024-09-21T10:06:29.971 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:06:29.971 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 60 16 103070 103054 107.328 68.4375 0.00930493 0.00931215 2024-09-21T10:06:29.982 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:30.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.024 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:30.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:30.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:30.167 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:30.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:30.269 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:30.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.325 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:30.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:30.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:30.463 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:30.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:30.477 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:06:30.477 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:30.522 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:06:30.523 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:30.561 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:06:30.562 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:30.565 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:30.595 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:30.606 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:06:30.606 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:30.642 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:06:30.643 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:06:30.673 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:06:30.673 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:30.697 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:30.712 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:06:30.712 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:30.728 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:30.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:30.745 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:06:30.746 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:30.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:30.882 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:30.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:30.971 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 61 16 104102 104086 106.626 64.5 0.00737053 0.00937355 2024-09-21T10:06:30.984 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:31.119 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:31.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:31.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:31.255 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:31.333 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:31.367 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:31 smithi032 ceph-mon[105748]: pgmap v343: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:06:31.367 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:31 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3477640268' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:31.367 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:31 smithi032 ceph-mon[101299]: pgmap v343: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:06:31.367 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:31 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3477640268' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:31.436 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:31.475 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:31.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:31.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:31 smithi045 ceph-mon[95976]: pgmap v343: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:06:31.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:31 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3477640268' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:31.577 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:31.614 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:31.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:31.646 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:06:31.647 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:06:31.716 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:31.753 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:31.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:31.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:31.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:31.971 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 62 16 105200 105184 106.013 68.625 0.00642965 0.00942789 2024-09-21T10:06:31.997 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:32.034 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:32.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:32.136 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:32.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:32.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:32.228 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:32.271 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:32.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:32.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:32.413 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:32.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.455 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:32.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:32.556 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:32.588 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:32.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:32.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:06:32.738 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:32.745 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:32.843 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:32.890 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:32.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:32.972 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 63 16 106320 106304 105.44 70 0.0160862 0.0094787 2024-09-21T10:06:32.992 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:33.027 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:33.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:33.129 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:33.176 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:33.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:33.262 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:33 smithi032 ceph-mon[105748]: pgmap v344: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:06:33.264 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:33 smithi032 ceph-mon[101299]: pgmap v344: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:06:33.278 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:33.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:33.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:33.414 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:33.451 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:33.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:33 smithi045 ceph-mon[95976]: pgmap v344: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:06:33.553 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:33.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:33.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:33.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:33.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.735 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:33.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:33.836 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:33.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.874 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:33.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:33.972 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 64 16 107455 107439 104.901 70.9375 0.00679371 0.00952806 2024-09-21T10:06:33.975 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:33.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:06:34.003 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:34.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:34.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:34.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.166 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:34.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:34.267 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:34.324 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:34.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:34.415 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:34.416 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:34.425 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:34.470 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:34.572 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:34.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:34.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:34.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.744 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:34.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:34.845 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:34.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:34.972 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 65 16 108513 108497 104.304 66.125 0.00386817 0.00957983 2024-09-21T10:06:34.985 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:35.007 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:35.109 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:35.141 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:35.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:35.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:35.282 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:35.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:35.384 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:35.419 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:35.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:35.521 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:35 smithi045 ceph-mon[95976]: pgmap v345: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-21T10:06:35.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:35.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:35.573 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:35 smithi032 ceph-mon[105748]: pgmap v345: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-21T10:06:35.573 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:35 smithi032 ceph-mon[101299]: pgmap v345: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-21T10:06:35.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:35.693 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:35.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:35.795 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:35.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.840 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:35.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:35.941 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:35.974 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 66 16 109550 109534 103.703 64.8125 0.0230579 0.00963774 2024-09-21T10:06:35.977 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:35.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:36.079 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.129 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:36.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:36.135 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:36.139 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:36.230 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:36.266 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:36.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:36.368 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:36.409 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:36.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:36.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:36.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:36.652 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:36.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.689 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:36.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:36.790 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:36.832 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:36.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:36.934 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:36.974 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 67 16 110582 110566 103.118 64.5 0.00972618 0.00969236 2024-09-21T10:06:36.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.976 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:36.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:37.077 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:37.117 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:37.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:37.219 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:37.234 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:06:37.239 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:06:37.262 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:37.364 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:37.383 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:37 smithi032 ceph-mon[101299]: pgmap v346: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-21T10:06:37.384 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:37 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4251920071' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:06:37.384 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:37 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:06:37.384 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:37 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1796072788' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:06:37.384 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:37 smithi032 ceph-mon[105748]: pgmap v346: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-21T10:06:37.384 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:37 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4251920071' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:06:37.384 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:37 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:06:37.384 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:37 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1796072788' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:06:37.404 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:37.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:37.506 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:37 smithi045 ceph-mon[95976]: pgmap v346: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-21T10:06:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:37 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4251920071' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:06:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:37 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:06:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:37 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1796072788' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:06:37.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:37.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:37.642 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:37.675 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:37.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.819 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:37.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:37.920 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:37.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.946 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:37.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:37.974 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 68 16 111660 111644 102.592 67.375 0.00962379 0.00974216 2024-09-21T10:06:38.047 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:38.076 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:38.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:38.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:38.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:38.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:38.310 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:38.352 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:38.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:38.454 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:38 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:38 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:06:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:38 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1796072788' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:06:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:38 smithi032 ceph-mon[105748]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T10:06:38.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:38 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:38.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:38 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:06:38.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:38 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1796072788' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:06:38.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:38 smithi032 ceph-mon[101299]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T10:06:38.485 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:38.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:38 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:38 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:06:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:38 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1796072788' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:06:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:38 smithi045 ceph-mon[95976]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T10:06:38.561 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:06:38.587 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:38.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.633 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:38.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:38.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:38.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:38.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:38.878 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:38.917 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:38.974 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 69 16 112752 112736 102.094 68.25 0.0107129 0.00978953 2024-09-21T10:06:39.018 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:39.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.088 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:39.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:39.189 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:39.227 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:39.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:39.306 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:39 smithi032 ceph-mon[101299]: pgmap v348: 89 pgs: 89 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:39.307 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:39 smithi032 ceph-mon[105748]: pgmap v348: 89 pgs: 89 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:39.328 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:39.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:39.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:39.463 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:39.509 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:39.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:39.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:39 smithi045 ceph-mon[95976]: pgmap v348: 89 pgs: 89 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:39.588 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:06:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:06:39] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.51.0" 2024-09-21T10:06:39.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:39.616 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:39.657 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:39.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:39.759 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:39.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.799 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:39.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:39.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:39.938 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:39.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:39.974 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 70 16 113810 113794 101.581 66.125 0.00999233 0.00983977 2024-09-21T10:06:40.040 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:40.076 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:40.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:40.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:40.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:40.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:40.327 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:40.366 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:40.468 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:40.504 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:40.606 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:40.637 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:40.739 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.773 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:40.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:40.874 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:40.915 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:40.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:40.975 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 71 16 114896 114880 101.106 67.875 0.0108054 0.00988571 2024-09-21T10:06:41.017 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:41.053 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:41.155 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:41.193 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:41.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:41.295 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:41.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:41.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:41.434 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:41.472 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:41.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:41.501 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:41 smithi032 ceph-mon[101299]: pgmap v349: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:41.502 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:41 smithi032 ceph-mon[105748]: pgmap v349: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:41.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:41 smithi045 ceph-mon[95976]: pgmap v349: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:41.574 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:41.590 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:06:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:41.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:41.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.749 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:41.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:41.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:41.892 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:41.975 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 72 16 115951 115935 100.617 65.9375 0.0091609 0.0099341 2024-09-21T10:06:41.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:42.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:42.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:42.138 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:42.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.173 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:42.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:42.275 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:42.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.314 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:42.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:42.416 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:42.453 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:42.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:42.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:42.600 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:42.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:42.703 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:42.738 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:42.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:42.841 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:42.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:42.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:42.976 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 73 16 117027 117011 100.159 67.25 0.00814986 0.0099796 2024-09-21T10:06:42.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:43.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.026 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:43.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:43.128 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:43.162 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:43.264 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:43.301 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:43.404 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:43.454 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:43.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:43 smithi045 ceph-mon[95976]: pgmap v350: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:43.554 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:43 smithi032 ceph-mon[105748]: pgmap v350: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:43.554 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:43 smithi032 ceph-mon[101299]: pgmap v350: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:06:43.556 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:43.607 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:43.709 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:43.738 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:06:43.738 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:06:43.756 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:43.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:43.858 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:43.895 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:43.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:43.976 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 74 16 117958 117942 99.5913 58.1875 0.0213298 0.0100357 2024-09-21T10:06:43.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:44.035 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:44.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:44.166 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:44.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:44.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:44.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:44.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:44.411 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:44.464 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:44.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:44.566 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:44.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:44.707 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:44.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.746 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:44.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:44.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:44.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:44.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:44.976 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 75 16 118831 118815 98.991 54.5625 0.0231697 0.010097 2024-09-21T10:06:44.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:45.026 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:45.128 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:45.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.169 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:45.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:45.247 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:06:45.269 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:45.307 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:45.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:45.409 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:45.445 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:45.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:45 smithi032 ceph-mon[105748]: pgmap v351: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2024-09-21T10:06:45.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:45 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2251291539' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:06:45.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:45 smithi032 ceph-mon[101299]: pgmap v351: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2024-09-21T10:06:45.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:45 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2251291539' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:06:45.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:45 smithi045 ceph-mon[95976]: pgmap v351: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2024-09-21T10:06:45.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:45 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2251291539' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:06:45.547 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:45.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:45.686 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:06:45.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:45.766 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:45.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:45.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:45.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:45.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:45.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 76 16 119803 119787 98.486 60.75 0.0126167 0.0101489 2024-09-21T10:06:46.006 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:46.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:46.142 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:46.177 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:46.279 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:46.327 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:46.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:46.429 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:46.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:46.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:46 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2251291539' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:06:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:46 smithi045 ceph-mon[95976]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T10:06:46.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:46.601 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:46.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:46.645 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:46 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2251291539' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:06:46.645 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:46 smithi032 ceph-mon[101299]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T10:06:46.645 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:46 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2251291539' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:06:46.645 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:46 smithi032 ceph-mon[105748]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T10:06:46.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:46.743 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:46.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:46.838 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:06:46.838 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:46.845 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:46.877 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:06:46.877 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:46.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:46.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:46.910 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:06:46.910 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:46.942 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:06:46.943 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:46.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 77 16 120835 120819 98.0446 64.5 0.0183151 0.0101946 2024-09-21T10:06:46.981 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:06:46.981 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:06:46.983 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:47.013 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:06:47.013 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:47.026 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:47.043 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:06:47.044 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:47.091 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:06:47.091 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:47.128 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:47.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.169 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:47.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:47.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:47.309 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:47.373 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:06:47.411 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:47.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:47.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:47 smithi045 ceph-mon[95976]: pgmap v353: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:06:47.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:47 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3990510376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:47.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:47.587 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:47.656 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:47 smithi032 ceph-mon[101299]: pgmap v353: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:06:47.656 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:47 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3990510376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:47.657 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:47 smithi032 ceph-mon[105748]: pgmap v353: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:06:47.657 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:47 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3990510376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:47.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:47.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:47.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:47.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:47.869 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:47.971 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:47.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 78 16 121853 121837 97.6033 63.625 0.015958 0.0102407 2024-09-21T10:06:48.003 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:48.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:48.105 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:48.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.141 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:48.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:48.151 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:48.211 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:48.242 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:48.299 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:48.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:48.401 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.443 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:48.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:48.544 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:48.589 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:48.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:48.691 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:48.725 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:48.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:48.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:48.855 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:48.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:48.957 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:48.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 79 16 122766 122750 97.0901 57.0625 0.0196594 0.0102945 2024-09-21T10:06:48.990 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:49.092 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:49.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.136 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:49.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:49.237 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:49.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.275 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:49.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:49.376 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:49.412 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:49.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:49.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:49 smithi045 ceph-mon[95976]: pgmap v354: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:06:49.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:49.536 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:49 smithi032 ceph-mon[105748]: pgmap v354: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:06:49.536 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:49.536 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:49 smithi032 ceph-mon[101299]: pgmap v354: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:06:49.536 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:49.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:49.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:49.650 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:49.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:49.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:49.790 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:49.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.829 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:49.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:49.930 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:49.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:06:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:06:49] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.51.0" 2024-09-21T10:06:49.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:49.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:49.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:06:49.977237+0000 min lat: 0.00110358 max lat: 0.182943 avg lat: 0.0103414 2024-09-21T10:06:49.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:06:49.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 80 16 123748 123732 96.6437 61.375 0.00940292 0.0103414 2024-09-21T10:06:50.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:50.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:50.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:50.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:06:50.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:50.249 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:50.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:50.351 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:50.388 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:50.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:50.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:50.530 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:50.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:50.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:50.665 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:50.766 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:50.812 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:50.913 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:50.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.955 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:50.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:50.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 81 16 124754 124738 96.2267 62.875 0.0358989 0.0103872 2024-09-21T10:06:51.056 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:51.096 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:51.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:51.198 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:51.231 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:51.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:51.333 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:51.369 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:51.471 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:51.503 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:51.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:51 smithi045 ceph-mon[95976]: pgmap v355: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:06:51.605 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:51.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:51.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:51.678 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:51 smithi032 ceph-mon[105748]: pgmap v355: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:06:51.678 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:51 smithi032 ceph-mon[101299]: pgmap v355: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:06:51.741 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:51.784 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:51.885 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:51.892 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:51.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.915 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:51.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:51.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 82 16 125768 125752 95.8261 63.375 0.0293205 0.010431 2024-09-21T10:06:52.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:52.062 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:52.164 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:52.203 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:52.305 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:52.341 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:52.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:52.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:52.477 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:52.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:52.579 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:52.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:52.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:52.710 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:52.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:52.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:52.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:52.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:52.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 83 16 126756 126740 95.4155 61.75 0.0104633 0.0104755 2024-09-21T10:06:52.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:53.027 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:53.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:53.129 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:53.177 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:53.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:53.278 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:53.281 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:53 smithi032 ceph-mon[101299]: pgmap v356: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:06:53.320 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:53.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:53.422 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:53.462 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:53.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:53 smithi045 ceph-mon[95976]: pgmap v356: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:06:53.564 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:53.600 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:53.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:53.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:53.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:53 smithi032 ceph-mon[105748]: pgmap v356: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:06:53.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.749 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:53.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:53.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:26446 5.a 5:5eab8da1:::benchmark_data_smithi032_140417_object26445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:38.493729+0000", 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 75.379696088000003, 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16661439, 2024-09-21T10:06:53.885 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26446 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493729+0000", 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.886 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493729+0000", 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493730+0000", 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5200000000000002e-07 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493750+0000", 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9678999999999999e-05 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.887 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493760+0000", 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0258000000000001e-05 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493765+0000", 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9810000000000003e-06 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493833+0000", 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.86e-05 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.888 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.494024+0000", 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00019030699999999999 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.494169+0000", 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014491600000000001 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495490+0000", 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001321773 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.889 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.497526+0000", 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002035935 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.660221+0000", 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16269486399999999 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.660292+0000", 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0592000000000004e-05 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.890 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.660344+0000", 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2033000000000001e-05 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90171 5.b 5:d7bd192a:::benchmark_data_smithi032_140417_object90170:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.387561+0000", 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.485864368000001, 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.149472629, 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.891 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90171 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.387561+0000", 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.892 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.387561+0000", 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.387562+0000", 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3379999999999999e-06 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.387586+0000", 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3773000000000001e-05 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.893 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.387734+0000", 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000147548 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.387742+0000", 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4020000000000002e-06 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.387795+0000", 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2398000000000002e-05 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.894 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.387978+0000", 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018292499999999999 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.388104+0000", 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012604900000000001 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401134+0000", 2024-09-21T10:06:53.895 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013030557 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401169+0000", 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5275999999999999e-05 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401250+0000", 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0933999999999995e-05 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401299+0000", 2024-09-21T10:06:53.896 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8415999999999999e-05 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537034+0000", 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13573501299999999 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90177 5.b 5:de9c011b:::benchmark_data_smithi032_140417_object90176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.391451+0000", 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.481974749000003, 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14575861700000001, 2024-09-21T10:06:53.897 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90177 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391451+0000", 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.898 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391451+0000", 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391452+0000", 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4449999999999999e-06 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391484+0000", 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1778e-05 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.899 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391496+0000", 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1613e-05 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391503+0000", 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8310000000000001e-06 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391708+0000", 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00020415599999999999 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.900 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391849+0000", 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014095399999999999 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392049+0000", 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00020068899999999999 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401201+0000", 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0091522860000000008 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.901 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537087+0000", 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.135885587 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537100+0000", 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2621e-05 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537162+0000", 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1958000000000005e-05 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.902 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537209+0000", 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7698999999999999e-05 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90178 5.b 5:d97522ac:::benchmark_data_smithi032_140417_object90177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.391523+0000", 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.481902237, 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14582822600000001, 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.903 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90178 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391523+0000", 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391523+0000", 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.904 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391524+0000", 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3399999999999999e-07 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391545+0000", 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1250000000000002e-05 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391553+0000", 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5100000000000001e-06 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.905 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.391558+0000", 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5459999999999999e-06 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.394943+0000", 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003385398 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.395059+0000", 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011588 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.906 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.395185+0000", 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012525599999999999 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401234+0000", 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0060489000000000003 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537246+0000", 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13601248099999999 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.907 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537265+0000", 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8940000000000002e-05 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537306+0000", 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0982000000000002e-05 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537351+0000", 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5549e-05 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.908 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90181 5.8 5:1b94b670:::benchmark_data_smithi032_140417_object90180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.392375+0000", 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.481050033000002, 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14867080599999999, 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90181 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.909 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392375+0000", 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392375+0000", 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392376+0000", 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4499999999999995e-07 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.910 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392400+0000", 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.406e-05 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392411+0000", 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0856000000000001e-05 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392419+0000", 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2620000000000001e-06 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.911 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392471+0000", 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2479999999999999e-05 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392580+0000", 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000108514 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.392701+0000", 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000120982 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.912 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.400472+0000", 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0077715199999999996 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.400901+0000", 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00042850899999999999 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.540946+0000", 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14004576399999999 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.541001+0000", 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4870999999999997e-05 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.541046+0000", 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5043000000000002e-05 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90182 5.8 5:1ba94cac:::benchmark_data_smithi032_140417_object90181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:06:53.914 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.393051+0000", 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.480374402000002, 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14810646699999999, 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90182 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.915 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.393051+0000", 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.393051+0000", 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.393052+0000", 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3e-07 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.916 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.393073+0000", 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0886e-05 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.393083+0000", 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0535e-05 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.393090+0000", 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8460000000000002e-06 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.917 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.395678+0000", 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0025880629999999998 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.395785+0000", 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010665 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.395920+0000", 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013524699999999999 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401039+0000", 2024-09-21T10:06:53.918 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0051185700000000002 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401081+0000", 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2596999999999999e-05 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.541086+0000", 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14000478299999999 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.541127+0000", 2024-09-21T10:06:53.919 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0892000000000001e-05 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.541158+0000", 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0667999999999998e-05 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90189 5.8 5:17f68a22:::benchmark_data_smithi032_140417_object90188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.401156+0000", 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.472269568000002, 2024-09-21T10:06:53.920 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.170571483, 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90189 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401156+0000", 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.921 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401156+0000", 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401157+0000", 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3689999999999999e-06 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401186+0000", 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.8291000000000001e-05 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.922 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.923 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.923 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401197+0000", 2024-09-21T10:06:53.923 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1817999999999999e-05 2024-09-21T10:06:53.923 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:53.924 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.924 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.924 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.924 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401205+0000", 2024-09-21T10:06:53.924 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.199e-06 2024-09-21T10:06:53.924 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.924 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401272+0000", 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6995999999999993e-05 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537526+0000", 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13625469900000001 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537716+0000", 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018937500000000001 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.925 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.552778+0000", 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.015061830999999999 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.567174+0000", 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.014396048 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.571634+0000", 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0044601789999999999 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.926 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.571686+0000", 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2626000000000001e-05 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.571727+0000", 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1051999999999999e-05 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.927 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90193 5.a 5:5fe0d566:::benchmark_data_smithi032_140417_object90192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.401232+0000", 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.472193554999997, 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.146128707, 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90193 2024-09-21T10:06:53.928 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401232+0000", 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401232+0000", 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401232+0000", 2024-09-21T10:06:53.929 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5199999999999997e-07 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401257+0000", 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4054e-05 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401265+0000", 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7700000000000007e-06 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401271+0000", 2024-09-21T10:06:53.930 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.66e-06 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.536097+0000", 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13482586199999999 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.536243+0000", 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014625200000000001 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.536394+0000", 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015128 2024-09-21T10:06:53.931 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.546354+0000", 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0099595650000000001 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.546470+0000", 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011607000000000001 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.547254+0000", 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000783499 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.932 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.547301+0000", 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7029e-05 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.547361+0000", 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0114000000000001e-05 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90194 5.8 5:1c4e7117:::benchmark_data_smithi032_140417_object90193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.401379+0000", 2024-09-21T10:06:53.933 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.472046675999998, 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.17066425900000001, 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90194 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401379+0000", 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.934 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401379+0000", 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401379+0000", 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6400000000000002e-07 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.935 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.401415+0000", 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5358000000000003e-05 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.535240+0000", 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.133825468 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.535251+0000", 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1153e-05 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.936 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.541187+0000", 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0059351949999999999 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.541723+0000", 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00053682000000000003 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.541852+0000", 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012852499999999999 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.937 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.552830+0000", 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010978382 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.567179+0000", 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.014348553 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.571943+0000", 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0047638489999999997 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.572004+0000", 2024-09-21T10:06:53.938 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0804000000000003e-05 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.572043+0000", 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9487999999999999e-05 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44543.0:90177 5.b e122/112 ondisk, result = 0)", 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.402358+0000", 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.471067015000003, 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13499581099999999, 2024-09-21T10:06:53.939 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.402358+0000", 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.402358+0000", 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.402360+0000", 2024-09-21T10:06:53.940 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1230000000000001e-06 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.402364+0000", 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.498e-06 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.535073+0000", 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13270845100000001 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.535088+0000", 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5299999999999999e-05 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.941 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537071+0000", 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0019836229999999999 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537081+0000", 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0720000000000006e-06 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.537354+0000", 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00027374399999999997 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.942 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:99165 5.b 5:d2865efa:::benchmark_data_smithi032_140417_object99164:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.263729+0000", 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.609696714000002, 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14906639099999999, 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 99165 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.943 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.263729+0000", 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.263729+0000", 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.263730+0000", 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9299999999999997e-07 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.944 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.263746+0000", 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6875999999999999e-05 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.263755+0000", 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7399999999999993e-06 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.263760+0000", 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9409999999999998e-06 2024-09-21T10:06:53.945 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.263816+0000", 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5529000000000003e-05 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.263923+0000", 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010733100000000001 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:06:53.946 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.264041+0000", 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000117636 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.267280+0000", 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0032392269999999999 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.268057+0000", 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00077665600000000003 2024-09-21T10:06:53.947 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412711+0000", 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.144654172 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412757+0000", 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6644000000000003e-05 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.948 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412795+0000", 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.7846000000000003e-05 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:99167 5.b 5:d3dc5508:::benchmark_data_smithi032_140417_object99166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.269990+0000", 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.603435473000001, 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14688414899999999, 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.949 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.950 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.950 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:53.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:53.951 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 99167 2024-09-21T10:06:53.951 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.951 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.951 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.951 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.951 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.269990+0000", 2024-09-21T10:06:53.951 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.269990+0000", 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.269992+0000", 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.81e-06 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270018+0000", 2024-09-21T10:06:53.952 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6089e-05 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270027+0000", 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4779999999999997e-06 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270033+0000", 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3730000000000001e-06 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270086+0000", 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3066000000000003e-05 2024-09-21T10:06:53.953 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270172+0000", 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6454999999999998e-05 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270260+0000", 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7400999999999999e-05 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412978+0000", 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14271855 2024-09-21T10:06:53.954 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416733+0000", 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003754921 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416782+0000", 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8539999999999999e-05 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.955 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416834+0000", 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2645999999999997e-05 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416874+0000", 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9820000000000002e-05 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:99169 5.b 5:d6a9cf4a:::benchmark_data_smithi032_140417_object99168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:06:53.956 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.270983+0000", 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.602442761999999, 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.158702386, 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 99169 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270983+0000", 2024-09-21T10:06:53.957 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270983+0000", 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270984+0000", 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5900000000000005e-07 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.958 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271004+0000", 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0409999999999999e-05 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271015+0000", 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1334e-05 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271022+0000", 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8750000000000002e-06 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.959 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274116+0000", 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0030941089999999998 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274277+0000", 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016061800000000001 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274420+0000", 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014304899999999999 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.960 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.413031+0000", 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13861069000000001 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416739+0000", 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003708303 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.429496+0000", 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012757258 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.429553+0000", 2024-09-21T10:06:53.961 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6719999999999999e-05 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.429685+0000", 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013206099999999999 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:99170 5.b 5:d61691e6:::benchmark_data_smithi032_140417_object99169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.271159+0000", 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.602266668999999, 2024-09-21T10:06:53.962 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16325490100000001, 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 99170 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271159+0000", 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.963 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271159+0000", 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271160+0000", 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2099999999999995e-07 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271181+0000", 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1577999999999999e-05 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.964 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271191+0000", 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.9809999999999998e-06 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271197+0000", 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1500000000000004e-06 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.413060+0000", 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14186233000000001 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.965 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.413195+0000", 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013558099999999999 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.413327+0000", 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013218299999999999 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416947+0000", 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00361985 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.966 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.419223+0000", 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002275207 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.434307+0000", 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.015084428 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.434371+0000", 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4374000000000004e-05 2024-09-21T10:06:53.967 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.434414+0000", 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2417999999999999e-05 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44543.0:99165 5.b e124/112 ondisk, result = 0)", 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.271515+0000", 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.601910641, 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14128643699999999, 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.968 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271515+0000", 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271515+0000", 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271516+0000", 2024-09-21T10:06:53.969 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1000000000000001e-06 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271521+0000", 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6070000000000001e-06 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271532+0000", 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1751e-05 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271539+0000", 2024-09-21T10:06:53.970 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2469999999999997e-06 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412687+0000", 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.141148571 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412704+0000", 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6928000000000001e-05 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.971 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412801+0000", 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7232999999999996e-05 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:99173 5.8 5:139932b7:::benchmark_data_smithi032_140417_object99172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.271992+0000", 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.601433212, 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14066737100000001, 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.972 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 99173 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271992+0000", 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271992+0000", 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.973 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.271993+0000", 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8100000000000001e-07 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.272015+0000", 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1824e-05 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.272027+0000", 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1925e-05 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.974 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.272035+0000", 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6210000000000004e-06 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.272089+0000", 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4821999999999998e-05 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.272221+0000", 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013120699999999999 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.975 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.272325+0000", 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010383900000000001 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276517+0000", 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0041926619999999998 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412510+0000", 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13599276199999999 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.976 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.977 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412549+0000", 2024-09-21T10:06:53.977 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8994e-05 2024-09-21T10:06:53.977 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.977 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.977 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.977 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412609+0000", 2024-09-21T10:06:53.977 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9721e-05 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412660+0000", 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1013000000000001e-05 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44543.0:99167 5.b e124/112 ondisk, result = 0)", 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.275548+0000", 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.597877896, 2024-09-21T10:06:53.978 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.137444763, 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275548+0000", 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275548+0000", 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.979 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275548+0000", 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2099999999999998e-07 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275550+0000", 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.43e-06 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275557+0000", 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5130000000000002e-06 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.980 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275562+0000", 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.826e-06 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412823+0000", 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13726091900000001 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412829+0000", 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0000000000000002e-06 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.981 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412992+0000", 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000163754 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:99175 5.4 e124/112 5:24cef222:::benchmark_data_smithi032_140417_object99174:head v 124'6276, mlcod=124'6274)", 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.276122+0000", 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.597303226000001, 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.139389969, 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.982 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276122+0000", 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276122+0000", 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276123+0000", 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0900000000000001e-07 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.983 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276199+0000", 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6462000000000001e-05 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276209+0000", 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3780000000000007e-06 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276215+0000", 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2319999999999998e-06 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.984 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276280+0000", 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5067999999999994e-05 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276297+0000", 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7312e-05 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.415444+0000", 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13914638500000001 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.415512+0000", 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8423000000000005e-05 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44543.0:99173 5.8 e124/112 ondisk, result = 0)", 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.276588+0000", 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.596837226000002, 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13592955500000001, 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.986 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276588+0000", 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276588+0000", 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276589+0000", 2024-09-21T10:06:53.987 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3500000000000005e-07 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276593+0000", 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2150000000000003e-06 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276604+0000", 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0324e-05 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.988 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.411597+0000", 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.134993055 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412495+0000", 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00089815700000000001 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412504+0000", 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9670000000000007e-06 2024-09-21T10:06:53.989 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412518+0000", 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3902e-05 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:99181 5.6 5:661f7f27:::benchmark_data_smithi032_140417_object99180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.277027+0000", 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.596398482000001, 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13956070100000001, 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:06:53.990 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 99181 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.277027+0000", 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:06:53.991 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.277027+0000", 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.277028+0000", 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.153e-06 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.277046+0000", 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8051e-05 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.411648+0000", 2024-09-21T10:06:53.992 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.134601687 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.411674+0000", 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6026000000000001e-05 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.411759+0000", 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4748999999999998e-05 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.411910+0000", 2024-09-21T10:06:53.993 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015100000000000001 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 5,7", 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412054+0000", 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014420599999999999 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.415615+0000", 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0035613519999999998 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416020+0000", 2024-09-21T10:06:53.994 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00040500200000000001 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416476+0000", 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00045594800000000002 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416538+0000", 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1576999999999997e-05 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416588+0000", 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9950000000000001e-05 2024-09-21T10:06:53.995 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.996 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.996 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.996 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:06:53.996 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:06:53.996 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:06:53.996 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 84 16 127726 127710 95.0013 60.625 0.0120315 0.0105208 2024-09-21T10:06:54.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:54.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:54.198 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:54.240 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:54.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:54.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:54.378 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:54.480 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:54.516 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:54.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:54.617 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:54.660 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:54.762 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:54.797 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:54.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:54.899 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:54.936 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:54.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:54.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 85 16 128709 128693 94.6064 61.4375 0.0219829 0.0105651 2024-09-21T10:06:55.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:55.084 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:55.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:55.185 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:55.218 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:55.320 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:55.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.357 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:55.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:55.417 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:55 smithi032 ceph-mon[105748]: pgmap v357: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-21T10:06:55.417 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:55 smithi032 ceph-mon[101299]: pgmap v357: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-21T10:06:55.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:55.498 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:55.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:55.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:55 smithi045 ceph-mon[95976]: pgmap v357: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-21T10:06:55.612 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:55.642 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:55.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:55.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:55.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:55.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:55.891 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:55.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:55.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:55.978 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 86 16 129635 129619 94.1793 57.875 0.0171259 0.0106129 2024-09-21T10:06:56.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:56.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.070 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:56.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:56.170 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:56.218 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:56.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:56.299 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:06:56.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:56.359 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:56.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:56.461 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:56.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:56.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:56.598 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:56.639 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:56.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:56.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2328449121' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:56.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:56 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:56.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2328449121' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:56.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:56 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:56.740 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:56.783 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:56.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:56.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2328449121' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:56.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:56 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:56.885 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:56.903 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:06:56.914 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:56.979 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 87 16 130616 130600 93.8015 61.3125 0.018893 0.0106563 2024-09-21T10:06:57.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:57.064 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:57.165 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.203 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:57.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:57.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:57.347 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:57.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:57.449 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[105748]: pgmap v358: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[105748]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[101299]: pgmap v358: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:57.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:57 smithi032 ceph-mon[101299]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T10:06:57.490 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:57.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:57.592 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:57.632 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:57.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:57.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:57.767 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:57.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:57 smithi045 ceph-mon[95976]: pgmap v358: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2024-09-21T10:06:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:57 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:06:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:57 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:06:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:57 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:57 smithi045 ceph-mon[95976]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T10:06:57.868 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:57.904 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:57.979 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 88 16 131564 131548 93.4088 59.25 0.0107852 0.0107004 2024-09-21T10:06:58.006 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.045 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:58.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:58.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:58.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:58.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:58.212 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:58.276 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:58.311 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:58.315 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:06:58.395 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:58 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3644595835' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:06:58.395 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:58 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:06:58.395 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:58 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3644595835' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:06:58.396 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:58 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:06:58.413 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:58.453 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:58.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:58.587 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:58.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:58.689 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:58.732 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:58.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:58.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:58 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3644595835' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:06:58.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:58 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:06:58.834 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:58.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:58.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:58.979 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 89 16 132522 132506 93.032 59.875 0.0108264 0.0107442 2024-09-21T10:06:58.985 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:59.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:59.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:59.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:59.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:59.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:59.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:59.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.295 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:59.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:59.396 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:59.438 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:59.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:59 smithi032 ceph-mon[105748]: pgmap v360: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2024-09-21T10:06:59.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:59 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:06:59.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:06:59 smithi032 ceph-mon[105748]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T10:06:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:59 smithi032 ceph-mon[101299]: pgmap v360: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2024-09-21T10:06:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:59 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:06:59.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:06:59 smithi032 ceph-mon[101299]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T10:06:59.540 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:59.574 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:59.676 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:59.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:06:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:59.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:59 smithi045 ceph-mon[95976]: pgmap v360: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2024-09-21T10:06:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:59 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:06:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:06:59 smithi045 ceph-mon[95976]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T10:06:59.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:59.849 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:06:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:06:59] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.51.0" 2024-09-21T10:06:59.852 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:59.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:59.954 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:59.979 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:06:59.979 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total time run: 90.0062 2024-09-21T10:06:59.979 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total writes made: 133388 2024-09-21T10:06:59.979 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Write size: 65536 2024-09-21T10:06:59.979 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object size: 65536 2024-09-21T10:06:59.979 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Bandwidth (MB/sec): 92.6242 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Bandwidth: 40.2421 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max bandwidth (MB/sec): 230.75 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min bandwidth (MB/sec): 54.5625 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average IOPS: 1481 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev IOPS: 643.874 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max IOPS: 3692 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min IOPS: 873 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average Latency(s): 0.010792 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Latency(s): 0.00853315 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max latency(s): 0.182943 2024-09-21T10:06:59.980 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min latency(s): 0.00110358 2024-09-21T10:06:59.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:59.984 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:07:00.081 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:00.123 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:00.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:00.224 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:00.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:00.361 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:00.380 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:07:00.381 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:00.392 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:00.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:00.415 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:07:00.416 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:00.459 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:07:00.459 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:00.493 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:00.495 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:07:00.495 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:00.522 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:00.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:00.527 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:07:00.527 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:07:00.561 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:07:00.562 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:00.598 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:07:00.598 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:00.624 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:00.634 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:07:00.634 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:00.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.652 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:00.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:00.753 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:00.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:00.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:00.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:07:00.896 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:00.941 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 7 1 2024-09-21T10:07:00.963 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:01.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:01.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:01.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:01.250 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:01.291 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:01.393 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:01.433 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:01.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:01.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:01 smithi032 ceph-mon[101299]: pgmap v362: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:07:01.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:01 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2578825909' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:01.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:01 smithi032 ceph-mon[105748]: pgmap v362: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:07:01.460 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:01 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2578825909' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:01.535 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:01.575 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:01.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:01.676 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:01.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:01.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:01 smithi045 ceph-mon[95976]: pgmap v362: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:07:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:01 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2578825909' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:01.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:01.848 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:01.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:01.950 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:01.988 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:01.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:02.090 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:02.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.121 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:02.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:02.221 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:02.253 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:02.355 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:02.398 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:02.499 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:02.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.534 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:02.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:02.634 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:02.670 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:02.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:02.772 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:02.806 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:02.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:02.907 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:02.943 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:02.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:03.044 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:03.083 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:03.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.215 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:03.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:03.316 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:03.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:03.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:03.438 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:03 smithi032 ceph-mon[101299]: pgmap v363: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 830 op/s 2024-09-21T10:07:03.438 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:03 smithi032 ceph-mon[105748]: pgmap v363: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 830 op/s 2024-09-21T10:07:03.465 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:03.514 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:03.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:03.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:03.644 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:03.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:03.681 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:07:03.681 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:07:03.747 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:03.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.780 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:03.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:03.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:03 smithi045 ceph-mon[95976]: pgmap v363: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 830 op/s 2024-09-21T10:07:03.881 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:03.915 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:03.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:03.977 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:03.977 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:04.017 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:04.049 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:04.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:04.151 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:04.188 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:04.290 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:04.325 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:04.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:04.427 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:04.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.464 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:04.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:04.564 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.600 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:04.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:04.701 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:04.736 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:04.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:04.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:04.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:04.796 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:04.796 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:04.796 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:04.796 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:04.831 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:07:04.838 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:04.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:07:04.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.871 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:04.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:04.972 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:05.003 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:05.104 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:05.146 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:05.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:05.248 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:05.280 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:05.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:05.382 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:05.424 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:05.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:05.525 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:05.573 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:05.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:05.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:05 smithi032 ceph-mon[105748]: pgmap v364: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 44 MiB/s wr, 931 op/s 2024-09-21T10:07:05.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:05 smithi032 ceph-mon[101299]: pgmap v364: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 44 MiB/s wr, 931 op/s 2024-09-21T10:07:05.674 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:05.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:05.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:05 smithi045 ceph-mon[95976]: pgmap v364: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 44 MiB/s wr, 931 op/s 2024-09-21T10:07:05.812 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.848 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:05.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:05.949 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:05.992 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:06.093 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:06.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.134 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:06.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:06.235 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:06.265 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:06.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:06.366 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:06.409 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:06.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:06.511 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:06.546 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:06.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:06.648 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.690 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:06.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:06.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:06.836 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:06.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:06.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:06.973 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:06.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:07.076 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:07.098 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:07.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:07.200 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:07.231 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:07.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:07.332 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:07.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.389 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:07.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:07.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:07.522 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:07.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:07.624 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:07.659 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:07.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:07.673 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:07 smithi032 ceph-mon[101299]: pgmap v365: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 103 B/s rd, 37 MiB/s wr, 961 op/s 2024-09-21T10:07:07.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:07 smithi032 ceph-mon[105748]: pgmap v365: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 103 B/s rd, 37 MiB/s wr, 961 op/s 2024-09-21T10:07:07.760 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:07.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:07 smithi045 ceph-mon[95976]: pgmap v365: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 103 B/s rd, 37 MiB/s wr, 961 op/s 2024-09-21T10:07:07.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:07.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:07.903 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:07.945 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:08.047 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.085 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:08.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:08.164 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:08.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:08.231 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:08.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:08.332 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:08.374 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:08.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:08.445 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:08.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:08.530 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:08.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:08.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:08.669 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:08.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:08.771 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:08.816 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:08.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:08.918 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:08.952 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:08.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:09.054 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:09.097 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:09.199 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:09.237 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:09.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:09.339 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:09.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:09.481 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:09.509 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:07:09.512 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.7 to 1 (10000) 2024-09-21T10:07:09.519 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:09.620 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:09.654 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:09.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:09.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:09 smithi045 ceph-mon[95976]: pgmap v366: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 25 MiB/s wr, 946 op/s 2024-09-21T10:07:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2593806339' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:07:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:09 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:07:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2318105009' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:09 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:09.800 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:09.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:09.837 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:07:09] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.51.0" 2024-09-21T10:07:09.838 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[101299]: pgmap v366: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 25 MiB/s wr, 946 op/s 2024-09-21T10:07:09.839 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2593806339' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:07:09.839 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:07:09.839 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2318105009' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:09.839 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:09.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[105748]: pgmap v366: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 25 MiB/s wr, 946 op/s 2024-09-21T10:07:09.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2593806339' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:07:09.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:07:09.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2318105009' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:09.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:09 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:09.902 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:09.940 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:10.042 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:10.078 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:10.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:10.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:10.213 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:10.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:10.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:10.353 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:10.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:10.454 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:10.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:10.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:10.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:10.631 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:10.733 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:10.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.770 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:10.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:10.871 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:10.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:10.924 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:10 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:10.924 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:10 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:07:10.924 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:10 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:07:10.924 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:10 smithi032 ceph-mon[101299]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T10:07:10.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:10 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:10.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:10 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:07:10.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:10 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:07:10.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:10 smithi032 ceph-mon[105748]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T10:07:11.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:10 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:10 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:07:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:10 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:07:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:10 smithi045 ceph-mon[95976]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T10:07:11.039 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:11.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:11.141 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:11.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.186 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:11.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:11.287 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:11.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.323 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:11.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:11.424 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:11.472 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:11.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:11.573 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:11.608 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:11 smithi032 ceph-mon[105748]: pgmap v368: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 936 op/s 2024-09-21T10:07:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:11 smithi032 ceph-mon[101299]: pgmap v368: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 936 op/s 2024-09-21T10:07:11.710 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:11.746 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:11.769 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:07:11.769 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:07:11.848 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:11.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:11.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:11.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:12.024 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:12.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:11 smithi045 ceph-mon[95976]: pgmap v368: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 936 op/s 2024-09-21T10:07:12.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:12.158 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:12.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:12.259 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:12.301 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:12.403 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:12.440 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:12.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:12.542 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:12.582 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:12.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:12.684 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:12.726 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:12.828 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:12.862 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:12.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:12.964 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:13.000 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:13.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:13.102 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:13.134 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:13.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:13.235 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:13.282 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:13.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:13.384 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:13.412 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:13.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:13.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:13.551 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:13.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.689 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:13.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:13.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:13.824 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:13.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:13.836 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:13 smithi045 ceph-mon[95976]: pgmap v369: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 5.7 MiB/s wr, 932 op/s 2024-09-21T10:07:13.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:13 smithi032 ceph-mon[105748]: pgmap v369: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 5.7 MiB/s wr, 932 op/s 2024-09-21T10:07:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:13 smithi032 ceph-mon[101299]: pgmap v369: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 5.7 MiB/s wr, 932 op/s 2024-09-21T10:07:13.957 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:14.059 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:14.092 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:14.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:14.112 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:07:14.133 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:07:14.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:07:14.194 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:14.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:14.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:14.360 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:14.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:14.463 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:14.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:14.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:14.600 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:14.637 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:14.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:14.739 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:14.771 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:14.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:14.872 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:14.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:14.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:15.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:15.065 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:15.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:15.167 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:15.196 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:15.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:15.297 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:15.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:15.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:15.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:15.455 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:07:15.477 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:15.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:15.579 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:15.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:15.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:15.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:15 smithi032 ceph-mon[101299]: pgmap v370: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.4 MiB/s wr, 945 op/s 2024-09-21T10:07:15.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:15.711 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:15 smithi032 ceph-mon[105748]: pgmap v370: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.4 MiB/s wr, 945 op/s 2024-09-21T10:07:15.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:15.819 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:07:15.820 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:07:15.820 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:07:15.820 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:15.820 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:15.820 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:15.831 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:15.847 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:15.852 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:07:15.852 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:15.874 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:15.875 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:15.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:15.887 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:07:15.887 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:07:15.926 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:07:15.926 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:15.968 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:07:15.968 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:15.977 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:16.003 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:07:16.004 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:16.008 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:16.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:15 smithi045 ceph-mon[95976]: pgmap v370: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.4 MiB/s wr, 945 op/s 2024-09-21T10:07:16.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:16.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:16.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:16.250 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:16.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.293 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:16.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:16.394 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.435 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:16.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:16.536 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:16.561 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:16.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:16.662 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:16.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:16.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:16.801 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:16.840 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:16.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:16.942 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:16.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:16 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/522110204' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:16.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:16 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/522110204' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:16.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:16 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/522110204' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:17.087 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:17.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:17.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:17.271 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:17.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:17.407 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:17.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:17.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:17.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:17.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:17.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:17.687 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:17.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:17.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:17 smithi032 ceph-mon[105748]: pgmap v371: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-09-21T10:07:17.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:17 smithi032 ceph-mon[101299]: pgmap v371: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-09-21T10:07:17.789 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:17.825 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:17.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:17.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:17.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:17.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:18.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:17 smithi045 ceph-mon[95976]: pgmap v371: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-09-21T10:07:18.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:18.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:18.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:18.201 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:18.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:18.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:18.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:18.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:18.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:18.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:18.514 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:18.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:18.615 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:18.649 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:18.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:18.754 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:18.791 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:18.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:18.893 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:18.931 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:19.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:19.063 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:19.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:19.164 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:19.199 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:19.301 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:19.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.342 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:19.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:19.443 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:19.474 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:19.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:19.575 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:19.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:19.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:07:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:07:19] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.51.0" 2024-09-21T10:07:19.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:19 smithi032 ceph-mon[105748]: pgmap v372: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2024-09-21T10:07:19.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:19.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:19.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:19 smithi032 ceph-mon[101299]: pgmap v372: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2024-09-21T10:07:19.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:19.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:19.708 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:19.747 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:19.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:19 smithi045 ceph-mon[95976]: pgmap v372: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2024-09-21T10:07:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:19.849 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:19.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:19.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:19.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:20.035 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:20.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:20.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:20.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:20.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:20.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:20.315 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:20.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:20.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:20.453 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:20.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:20.474 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:07:20.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:20.588 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:20.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:20.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:20.745 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:20.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:20.876 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:20.978 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:21.018 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:21.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:21.119 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:21.119 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:21.167 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:21.269 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:21.302 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:21.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:21.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:21.441 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:21.543 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.579 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:21.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:21.680 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:21.685 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:21 smithi032 ceph-mon[105748]: pgmap v373: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-21T10:07:21.687 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:21 smithi032 ceph-mon[101299]: pgmap v373: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-21T10:07:21.718 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:21.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:21.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:21 smithi045 ceph-mon[95976]: pgmap v373: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-21T10:07:21.820 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:21.857 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:21.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:21.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:21.992 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:22.094 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.130 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:22.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:22.231 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.270 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:22.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:22.371 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:22.403 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:22.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:22.505 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:22.544 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:22.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:22.646 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:22.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.690 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:22.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:22.791 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:22.826 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:22.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:22.928 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.973 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:22.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:23.038 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:07:23.043 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:07:23.074 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:23.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.131 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:23.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:23.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.267 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:23.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:23.368 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:23.410 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:23.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:23.476 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:07:23.512 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:23.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:23.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.690 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:23.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:23 smithi045 ceph-mon[95976]: pgmap v374: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-09-21T10:07:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1262764286' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:23.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:23.834 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:23.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:23.917 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:23.937 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:23.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:23 smithi032 ceph-mon[105748]: pgmap v374: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-09-21T10:07:23.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1262764286' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:23 smithi032 ceph-mon[101299]: pgmap v374: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-09-21T10:07:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1262764286' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:23.985 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:23.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:24.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:24.122 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:24.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:24.224 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:24.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:24.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:24.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:24.397 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:24.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:24.499 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:24.525 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":374,"stamp":"2024-09-21T10:07:22.177030+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7577737213,"num_objects":115583,"num_object_clones":0,"num_object_copies":346749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":115583,"num_whiteouts":0,"num_read":6391,"num_read_kb":13846,"num_write":152346,"num_write_kb":8551874,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143405,"ondisk_log_size":143405,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22951320,"kb_used_data":22251992,"kb_used_omap":35,"kb_used_meta":699164,"kb_avail":727009896,"statfs":{"total":767960285184,"available":744458133504,"internally_reserved":0,"allocated":22786039808,"data_stored":22780647167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":715944654},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,6,4,5,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-659161088,"num_objects":-10058,"num_object_clones":0,"num_object_copies":-30174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10058,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004529"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504153+0000","last_change":"2024-09-21T10:03:53.843548+0000","last_active":"2024-09-21T10:07:09.504153+0000","last_peered":"2024-09-21T10:07:09.504153+0000","last_clean":"2024-09-21T10:07:09.504153+0000","last_became_active":"2024-09-21T10:03:53.843182+0000","last_became_peered":"2024-09-21T10:03:53.843182+0000","last_unstale":"2024-09-21T10:07:09.504153+0000","last_undegraded":"2024-09-21T10:07:09.504153+0000","last_fullsized":"2024-09-21T10:07:09.504153+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:09.464209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.326308+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:06:58.326308+0000","last_peered":"2024-09-21T10:06:58.326308+0000","last_clean":"2024-09-21T10:06:58.326308+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:06:58.326308+0000","last_undegraded":"2024-09-21T10:06:58.326308+0000","last_fullsized":"2024-09-21T10:06:58.326308+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":92,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.508328+0000","last_change":"2024-09-21T10:03:53.839987+0000","last_active":"2024-09-21T10:07:09.508328+0000","last_peered":"2024-09-21T10:07:09.508328+0000","last_clean":"2024-09-21T10:07:09.508328+0000","last_became_active":"2024-09-21T10:03:53.839664+0000","last_became_peered":"2024-09-21T10:03:53.839664+0000","last_unstale":"2024-09-21T10:07:09.508328+0000","last_undegraded":"2024-09-21T10:07:09.508328+0000","last_fullsized":"2024-09-21T10:07:09.508328+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:51.726392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":104,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.521068+0000","last_change":"2024-09-21T10:03:53.837496+0000","last_active":"2024-09-21T10:07:09.521068+0000","last_peered":"2024-09-21T10:07:09.521068+0000","last_clean":"2024-09-21T10:07:09.521068+0000","last_became_active":"2024-09-21T10:03:53.837166+0000","last_became_peered":"2024-09-21T10:03:53.837166+0000","last_unstale":"2024-09-21T10:07:09.521068+0000","last_undegraded":"2024-09-21T10:07:09.521068+0000","last_fullsized":"2024-09-21T10:07:09.521068+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:45.795571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":106,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.519850+0000","last_change":"2024-09-21T10:03:53.844990+0000","last_active":"2024-09-21T10:07:09.519850+0000","last_peered":"2024-09-21T10:07:09.519850+0000","last_clean":"2024-09-21T10:07:09.519850+0000","last_became_active":"2024-09-21T10:03:53.844664+0000","last_became_peered":"2024-09-21T10:03:53.844664+0000","last_unstale":"2024-09-21T10:07:09.519850+0000","last_undegraded":"2024-09-21T10:07:09.519850+0000","last_fullsized":"2024-09-21T10:07:09.519850+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:01.031522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.326038+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:06:58.326038+0000","last_peered":"2024-09-21T10:06:58.326038+0000","last_clean":"2024-09-21T10:06:58.326038+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:06:58.326038+0000","last_undegraded":"2024-09-21T10:06:58.326038+0000","last_fullsized":"2024-09-21T10:06:58.326038+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":114,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504910+0000","last_change":"2024-09-21T10:03:53.845785+0000","last_active":"2024-09-21T10:07:09.504910+0000","last_peered":"2024-09-21T10:07:09.504910+0000","last_clean":"2024-09-21T10:07:09.504910+0000","last_became_active":"2024-09-21T10:03:53.845303+0000","last_became_peered":"2024-09-21T10:03:53.845303+0000","last_unstale":"2024-09-21T10:07:09.504910+0000","last_undegraded":"2024-09-21T10:07:09.504910+0000","last_fullsized":"2024-09-21T10:07:09.504910+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:04.976007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":101,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.320729+0000","last_change":"2024-09-21T10:02:12.838786+0000","last_active":"2024-09-21T10:06:58.320729+0000","last_peered":"2024-09-21T10:06:58.320729+0000","last_clean":"2024-09-21T10:06:58.320729+0000","last_became_active":"2024-09-21T10:02:12.838242+0000","last_became_peered":"2024-09-21T10:02:12.838242+0000","last_unstale":"2024-09-21T10:06:58.320729+0000","last_undegraded":"2024-09-21T10:06:58.320729+0000","last_fullsized":"2024-09-21T10:06:58.320729+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:31.071136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"129'9439","reported_seq":9465,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:18.977083+0000","last_change":"2024-09-21T10:05:19.211627+0000","last_active":"2024-09-21T10:07:18.977083+0000","last_peered":"2024-09-21T10:07:18.977083+0000","last_clean":"2024-09-21T10:07:18.977083+0000","last_became_active":"2024-09-21T10:05:19.211261+0000","last_became_peered":"2024-09-21T10:05:19.211261+0000","last_unstale":"2024-09-21T10:07:18.977083+0000","last_undegraded":"2024-09-21T10:07:18.977083+0000","last_fullsized":"2024-09-21T10:07:18.977083+0000","mapping_epoch":112,"log_start":"117'800","ondisk_log_start":"117'800","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8639,"log_dups_size":0,"ondisk_log_size":8639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:10.360788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":475332608,"num_objects":7253,"num_object_clones":0,"num_object_copies":21759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9439,"num_write_kb":534144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.326576+0000","last_change":"2024-09-21T10:02:12.840334+0000","last_active":"2024-09-21T10:06:58.326576+0000","last_peered":"2024-09-21T10:06:58.326576+0000","last_clean":"2024-09-21T10:06:58.326576+0000","last_became_active":"2024-09-21T10:02:12.839928+0000","last_became_peered":"2024-09-21T10:02:12.839928+0000","last_unstale":"2024-09-21T10:06:58.326576+0000","last_undegraded":"2024-09-21T10:06:58.326576+0000","last_fullsized":"2024-09-21T10:06:58.326576+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:38.593644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.323803+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T10:06:58.323803+0000","last_peered":"2024-09-21T10:06:58.323803+0000","last_clean":"2024-09-21T10:06:58.323803+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T10:06:58.323803+0000","last_undegraded":"2024-09-21T10:06:58.323803+0000","last_fullsized":"2024-09-21T10:06:58.323803+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"129'9354","reported_seq":9399,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:17.989593+0000","last_change":"2024-09-21T10:05:19.213525+0000","last_active":"2024-09-21T10:07:17.989593+0000","last_peered":"2024-09-21T10:07:17.989593+0000","last_clean":"2024-09-21T10:07:17.989593+0000","last_became_active":"2024-09-21T10:05:19.212434+0000","last_became_peered":"2024-09-21T10:05:19.212434+0000","last_unstale":"2024-09-21T10:07:17.989593+0000","last_undegraded":"2024-09-21T10:07:17.989593+0000","last_fullsized":"2024-09-21T10:07:17.989593+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9354,"log_dups_size":0,"ondisk_log_size":9354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:35.483451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474087424,"num_objects":7234,"num_object_clones":0,"num_object_copies":21702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9354,"num_write_kb":530816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.332932+0000","last_change":"2024-09-21T10:02:46.662169+0000","last_active":"2024-09-21T10:06:58.332932+0000","last_peered":"2024-09-21T10:06:58.332932+0000","last_clean":"2024-09-21T10:06:58.332932+0000","last_became_active":"2024-09-21T10:02:46.661721+0000","last_became_peered":"2024-09-21T10:02:46.661721+0000","last_unstale":"2024-09-21T10:06:58.332932+0000","last_undegraded":"2024-09-21T10:06:58.332932+0000","last_fullsized":"2024-09-21T10:06:58.332932+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:10.414152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.328154+0000","last_change":"2024-09-21T10:03:53.840738+0000","last_active":"2024-09-21T10:06:58.328154+0000","last_peered":"2024-09-21T10:06:58.328154+0000","last_clean":"2024-09-21T10:06:58.328154+0000","last_became_active":"2024-09-21T10:03:53.840363+0000","last_became_peered":"2024-09-21T10:03:53.840363+0000","last_unstale":"2024-09-21T10:06:58.328154+0000","last_undegraded":"2024-09-21T10:06:58.328154+0000","last_fullsized":"2024-09-21T10:06:58.328154+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:26.371181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"129'9398","reported_seq":9443,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:18.006275+0000","last_change":"2024-09-21T10:05:19.213181+0000","last_active":"2024-09-21T10:07:18.006275+0000","last_peered":"2024-09-21T10:07:18.006275+0000","last_clean":"2024-09-21T10:07:18.006275+0000","last_became_active":"2024-09-21T10:05:19.212895+0000","last_became_peered":"2024-09-21T10:05:19.212895+0000","last_unstale":"2024-09-21T10:07:18.006275+0000","last_undegraded":"2024-09-21T10:07:18.006275+0000","last_fullsized":"2024-09-21T10:07:18.006275+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9398,"log_dups_size":0,"ondisk_log_size":9398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:40:31.162520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477233152,"num_objects":7282,"num_object_clones":0,"num_object_copies":21846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9398,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":79,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.325827+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T10:06:58.325827+0000","last_peered":"2024-09-21T10:06:58.325827+0000","last_clean":"2024-09-21T10:06:58.325827+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T10:06:58.325827+0000","last_undegraded":"2024-09-21T10:06:58.325827+0000","last_fullsized":"2024-09-21T10:06:58.325827+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.322237+0000","last_change":"2024-09-21T10:02:46.663127+0000","last_active":"2024-09-21T10:06:58.322237+0000","last_peered":"2024-09-21T10:06:58.322237+0000","last_clean":"2024-09-21T10:06:58.322237+0000","last_became_active":"2024-09-21T10:02:46.662791+0000","last_became_peered":"2024-09-21T10:02:46.662791+0000","last_unstale":"2024-09-21T10:06:58.322237+0000","last_undegraded":"2024-09-21T10:06:58.322237+0000","last_fullsized":"2024-09-21T10:06:58.322237+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:59.990122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"129'9322","reported_seq":9367,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:18.022511+0000","last_change":"2024-09-21T10:05:19.212279+0000","last_active":"2024-09-21T10:07:18.022511+0000","last_peered":"2024-09-21T10:07:18.022511+0000","last_clean":"2024-09-21T10:07:18.022511+0000","last_became_active":"2024-09-21T10:05:19.211960+0000","last_became_peered":"2024-09-21T10:05:19.211960+0000","last_unstale":"2024-09-21T10:07:18.022511+0000","last_undegraded":"2024-09-21T10:07:18.022511+0000","last_fullsized":"2024-09-21T10:07:18.022511+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9322,"log_dups_size":0,"ondisk_log_size":9322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:40.251441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":483131392,"num_objects":7372,"num_object_clones":0,"num_object_copies":22116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9322,"num_write_kb":534208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":93,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.323914+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:06:58.323914+0000","last_peered":"2024-09-21T10:06:58.323914+0000","last_clean":"2024-09-21T10:06:58.323914+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:06:58.323914+0000","last_undegraded":"2024-09-21T10:06:58.323914+0000","last_fullsized":"2024-09-21T10:06:58.323914+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":106,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.521500+0000","last_change":"2024-09-21T10:02:46.662651+0000","last_active":"2024-09-21T10:07:09.521500+0000","last_peered":"2024-09-21T10:07:09.521500+0000","last_clean":"2024-09-21T10:07:09.521500+0000","last_became_active":"2024-09-21T10:02:46.662304+0000","last_became_peered":"2024-09-21T10:02:46.662304+0000","last_unstale":"2024-09-21T10:07:09.521500+0000","last_undegraded":"2024-09-21T10:07:09.521500+0000","last_fullsized":"2024-09-21T10:07:09.521500+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:06.746929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"129'9536","reported_seq":9582,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:18.001297+0000","last_change":"2024-09-21T10:05:19.211133+0000","last_active":"2024-09-21T10:07:18.001297+0000","last_peered":"2024-09-21T10:07:18.001297+0000","last_clean":"2024-09-21T10:07:18.001297+0000","last_became_active":"2024-09-21T10:05:19.210822+0000","last_became_peered":"2024-09-21T10:05:19.210822+0000","last_unstale":"2024-09-21T10:07:18.001297+0000","last_undegraded":"2024-09-21T10:07:18.001297+0000","last_fullsized":"2024-09-21T10:07:18.001297+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9536,"log_dups_size":0,"ondisk_log_size":9536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:02.399086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":482672664,"num_objects":7366,"num_object_clones":0,"num_object_copies":22098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7366,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9536,"num_write_kb":540801,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":112,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504049+0000","last_change":"2024-09-21T10:03:53.840511+0000","last_active":"2024-09-21T10:07:09.504049+0000","last_peered":"2024-09-21T10:07:09.504049+0000","last_clean":"2024-09-21T10:07:09.504049+0000","last_became_active":"2024-09-21T10:03:53.840136+0000","last_became_peered":"2024-09-21T10:03:53.840136+0000","last_unstale":"2024-09-21T10:07:09.504049+0000","last_undegraded":"2024-09-21T10:07:09.504049+0000","last_fullsized":"2024-09-21T10:07:09.504049+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:24.765990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":92,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.328629+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:06:58.328629+0000","last_peered":"2024-09-21T10:06:58.328629+0000","last_clean":"2024-09-21T10:06:58.328629+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:06:58.328629+0000","last_undegraded":"2024-09-21T10:06:58.328629+0000","last_fullsized":"2024-09-21T10:06:58.328629+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"129'9369","reported_seq":9414,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:19.482823+0000","last_change":"2024-09-21T10:05:19.214310+0000","last_active":"2024-09-21T10:07:19.482823+0000","last_peered":"2024-09-21T10:07:19.482823+0000","last_clean":"2024-09-21T10:07:19.482823+0000","last_became_active":"2024-09-21T10:05:19.213979+0000","last_became_peered":"2024-09-21T10:05:19.213979+0000","last_unstale":"2024-09-21T10:07:19.482823+0000","last_undegraded":"2024-09-21T10:07:19.482823+0000","last_fullsized":"2024-09-21T10:07:19.482823+0000","mapping_epoch":112,"log_start":"117'1200","ondisk_log_start":"117'1200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8169,"log_dups_size":0,"ondisk_log_size":8169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:05.291457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474021888,"num_objects":7233,"num_object_clones":0,"num_object_copies":21699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9369,"num_write_kb":531264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":93,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.322646+0000","last_change":"2024-09-21T10:02:46.663900+0000","last_active":"2024-09-21T10:06:58.322646+0000","last_peered":"2024-09-21T10:06:58.322646+0000","last_clean":"2024-09-21T10:06:58.322646+0000","last_became_active":"2024-09-21T10:02:46.663594+0000","last_became_peered":"2024-09-21T10:02:46.663594+0000","last_unstale":"2024-09-21T10:06:58.322646+0000","last_undegraded":"2024-09-21T10:06:58.322646+0000","last_fullsized":"2024-09-21T10:06:58.322646+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:24.140061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.327733+0000","last_change":"2024-09-21T10:03:19.765623+0000","last_active":"2024-09-21T10:06:58.327733+0000","last_peered":"2024-09-21T10:06:58.327733+0000","last_clean":"2024-09-21T10:06:58.327733+0000","last_became_active":"2024-09-21T10:03:19.765227+0000","last_became_peered":"2024-09-21T10:03:19.765227+0000","last_unstale":"2024-09-21T10:06:58.327733+0000","last_undegraded":"2024-09-21T10:06:58.327733+0000","last_fullsized":"2024-09-21T10:06:58.327733+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:52.227826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"129'9569","reported_seq":9595,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:19.255326+0000","last_change":"2024-09-21T10:05:19.212082+0000","last_active":"2024-09-21T10:07:19.255326+0000","last_peered":"2024-09-21T10:07:19.255326+0000","last_clean":"2024-09-21T10:07:19.255326+0000","last_became_active":"2024-09-21T10:05:19.211673+0000","last_became_peered":"2024-09-21T10:05:19.211673+0000","last_unstale":"2024-09-21T10:07:19.255326+0000","last_undegraded":"2024-09-21T10:07:19.255326+0000","last_fullsized":"2024-09-21T10:07:19.255326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9569,"log_dups_size":0,"ondisk_log_size":9569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:12.775587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":473628672,"num_objects":7227,"num_object_clones":0,"num_object_copies":21681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9569,"num_write_kb":537472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.328363+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:06:58.328363+0000","last_peered":"2024-09-21T10:06:58.328363+0000","last_clean":"2024-09-21T10:06:58.328363+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:06:58.328363+0000","last_undegraded":"2024-09-21T10:06:58.328363+0000","last_fullsized":"2024-09-21T10:06:58.328363+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":92,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.326677+0000","last_change":"2024-09-21T10:02:46.664014+0000","last_active":"2024-09-21T10:06:58.326677+0000","last_peered":"2024-09-21T10:06:58.326677+0000","last_clean":"2024-09-21T10:06:58.326677+0000","last_became_active":"2024-09-21T10:02:46.663640+0000","last_became_peered":"2024-09-21T10:02:46.663640+0000","last_unstale":"2024-09-21T10:06:58.326677+0000","last_undegraded":"2024-09-21T10:06:58.326677+0000","last_fullsized":"2024-09-21T10:06:58.326677+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:03.733462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"129'9463","reported_seq":9489,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:19.190456+0000","last_change":"2024-09-21T10:05:19.214684+0000","last_active":"2024-09-21T10:07:19.190456+0000","last_peered":"2024-09-21T10:07:19.190456+0000","last_clean":"2024-09-21T10:07:19.190456+0000","last_became_active":"2024-09-21T10:05:19.214356+0000","last_became_peered":"2024-09-21T10:05:19.214356+0000","last_unstale":"2024-09-21T10:07:19.190456+0000","last_undegraded":"2024-09-21T10:07:19.190456+0000","last_fullsized":"2024-09-21T10:07:19.190456+0000","mapping_epoch":112,"log_start":"117'2100","ondisk_log_start":"117'2100","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:04.876232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":473366528,"num_objects":7223,"num_object_clones":0,"num_object_copies":21669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9463,"num_write_kb":533952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":115,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.506620+0000","last_change":"2024-09-21T10:03:53.845620+0000","last_active":"2024-09-21T10:07:09.506620+0000","last_peered":"2024-09-21T10:07:09.506620+0000","last_clean":"2024-09-21T10:07:09.506620+0000","last_became_active":"2024-09-21T10:03:53.845290+0000","last_became_peered":"2024-09-21T10:03:53.845290+0000","last_unstale":"2024-09-21T10:07:09.506620+0000","last_undegraded":"2024-09-21T10:07:09.506620+0000","last_fullsized":"2024-09-21T10:07:09.506620+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:20.076623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":96,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.523789+0000","last_change":"2024-09-21T10:03:53.846606+0000","last_active":"2024-09-21T10:07:09.523789+0000","last_peered":"2024-09-21T10:07:09.523789+0000","last_clean":"2024-09-21T10:07:09.523789+0000","last_became_active":"2024-09-21T10:03:53.846245+0000","last_became_peered":"2024-09-21T10:03:53.846245+0000","last_unstale":"2024-09-21T10:07:09.523789+0000","last_undegraded":"2024-09-21T10:07:09.523789+0000","last_fullsized":"2024-09-21T10:07:09.523789+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:20.745470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"129'9533","reported_seq":9559,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:19.081531+0000","last_change":"2024-09-21T10:05:19.210796+0000","last_active":"2024-09-21T10:07:19.081531+0000","last_peered":"2024-09-21T10:07:19.081531+0000","last_clean":"2024-09-21T10:07:19.081531+0000","last_became_active":"2024-09-21T10:05:19.210466+0000","last_became_peered":"2024-09-21T10:05:19.210466+0000","last_unstale":"2024-09-21T10:07:19.081531+0000","last_undegraded":"2024-09-21T10:07:19.081531+0000","last_fullsized":"2024-09-21T10:07:19.081531+0000","mapping_epoch":112,"log_start":"117'900","ondisk_log_start":"117'900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8633,"log_dups_size":0,"ondisk_log_size":8633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:11.178437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474021888,"num_objects":7233,"num_object_clones":0,"num_object_copies":21699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9533,"num_write_kb":536512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":425,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.325645+0000","last_change":"2024-09-21T10:02:12.840225+0000","last_active":"2024-09-21T10:06:58.325645+0000","last_peered":"2024-09-21T10:06:58.325645+0000","last_clean":"2024-09-21T10:06:58.325645+0000","last_became_active":"2024-09-21T10:02:12.839813+0000","last_became_peered":"2024-09-21T10:02:12.839813+0000","last_unstale":"2024-09-21T10:06:58.325645+0000","last_undegraded":"2024-09-21T10:06:58.325645+0000","last_fullsized":"2024-09-21T10:06:58.325645+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:36.630881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":443,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.327499+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:06:58.327499+0000","last_peered":"2024-09-21T10:06:58.327499+0000","last_clean":"2024-09-21T10:06:58.327499+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:06:58.327499+0000","last_undegraded":"2024-09-21T10:06:58.327499+0000","last_fullsized":"2024-09-21T10:06:58.327499+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.503706+0000","last_change":"2024-09-21T10:03:53.843344+0000","last_active":"2024-09-21T10:07:09.503706+0000","last_peered":"2024-09-21T10:07:09.503706+0000","last_clean":"2024-09-21T10:07:09.503706+0000","last_became_active":"2024-09-21T10:03:53.842304+0000","last_became_peered":"2024-09-21T10:03:53.842304+0000","last_unstale":"2024-09-21T10:07:09.503706+0000","last_undegraded":"2024-09-21T10:07:09.503706+0000","last_fullsized":"2024-09-21T10:07:09.503706+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:31.221735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":78,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.125773+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:07:09.125773+0000","last_peered":"2024-09-21T10:07:09.125773+0000","last_clean":"2024-09-21T10:07:09.125773+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:07:09.125773+0000","last_undegraded":"2024-09-21T10:07:09.125773+0000","last_fullsized":"2024-09-21T10:07:09.125773+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"129'9529","reported_seq":9555,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:19.271117+0000","last_change":"2024-09-21T10:05:19.211638+0000","last_active":"2024-09-21T10:07:19.271117+0000","last_peered":"2024-09-21T10:07:19.271117+0000","last_clean":"2024-09-21T10:07:19.271117+0000","last_became_active":"2024-09-21T10:05:19.211301+0000","last_became_peered":"2024-09-21T10:05:19.211301+0000","last_unstale":"2024-09-21T10:07:19.271117+0000","last_undegraded":"2024-09-21T10:07:19.271117+0000","last_fullsized":"2024-09-21T10:07:19.271117+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9529,"log_dups_size":0,"ondisk_log_size":9529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:53.839733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479920128,"num_objects":7323,"num_object_clones":0,"num_object_copies":21969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9529,"num_write_kb":539264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":588,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.325621+0000","last_change":"2024-09-21T10:02:46.663394+0000","last_active":"2024-09-21T10:06:58.325621+0000","last_peered":"2024-09-21T10:06:58.325621+0000","last_clean":"2024-09-21T10:06:58.325621+0000","last_became_active":"2024-09-21T10:02:46.662879+0000","last_became_peered":"2024-09-21T10:02:46.662879+0000","last_unstale":"2024-09-21T10:06:58.325621+0000","last_undegraded":"2024-09-21T10:06:58.325621+0000","last_fullsized":"2024-09-21T10:06:58.325621+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.125795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":104,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.519010+0000","last_change":"2024-09-21T10:03:53.835329+0000","last_active":"2024-09-21T10:07:09.519010+0000","last_peered":"2024-09-21T10:07:09.519010+0000","last_clean":"2024-09-21T10:07:09.519010+0000","last_became_active":"2024-09-21T10:03:53.834934+0000","last_became_peered":"2024-09-21T10:03:53.834934+0000","last_unstale":"2024-09-21T10:07:09.519010+0000","last_undegraded":"2024-09-21T10:07:09.519010+0000","last_fullsized":"2024-09-21T10:07:09.519010+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:38.355817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":117,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.523564+0000","last_change":"2024-09-21T10:03:19.770016+0000","last_active":"2024-09-21T10:07:09.523564+0000","last_peered":"2024-09-21T10:07:09.523564+0000","last_clean":"2024-09-21T10:07:09.523564+0000","last_became_active":"2024-09-21T10:03:19.768866+0000","last_became_peered":"2024-09-21T10:03:19.768866+0000","last_unstale":"2024-09-21T10:07:09.523564+0000","last_undegraded":"2024-09-21T10:07:09.523564+0000","last_fullsized":"2024-09-21T10:07:09.523564+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:25.068940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"129'9335","reported_seq":9380,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:18.027743+0000","last_change":"2024-09-21T10:05:19.210830+0000","last_active":"2024-09-21T10:07:18.027743+0000","last_peered":"2024-09-21T10:07:18.027743+0000","last_clean":"2024-09-21T10:07:18.027743+0000","last_became_active":"2024-09-21T10:05:19.210525+0000","last_became_peered":"2024-09-21T10:05:19.210525+0000","last_unstale":"2024-09-21T10:07:18.027743+0000","last_undegraded":"2024-09-21T10:07:18.027743+0000","last_fullsized":"2024-09-21T10:07:18.027743+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9335,"log_dups_size":0,"ondisk_log_size":9335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:44.771575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":472711168,"num_objects":7213,"num_object_clones":0,"num_object_copies":21639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9335,"num_write_kb":529536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":612,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.514036+0000","last_change":"2024-09-21T10:03:53.840322+0000","last_active":"2024-09-21T10:07:09.514036+0000","last_peered":"2024-09-21T10:07:09.514036+0000","last_clean":"2024-09-21T10:07:09.514036+0000","last_became_active":"2024-09-21T10:03:53.839753+0000","last_became_peered":"2024-09-21T10:03:53.839753+0000","last_unstale":"2024-09-21T10:07:09.514036+0000","last_undegraded":"2024-09-21T10:07:09.514036+0000","last_fullsized":"2024-09-21T10:07:09.514036+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:26.364888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":94,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.517045+0000","last_change":"2024-09-21T10:03:19.765402+0000","last_active":"2024-09-21T10:07:09.517045+0000","last_peered":"2024-09-21T10:07:09.517045+0000","last_clean":"2024-09-21T10:07:09.517045+0000","last_became_active":"2024-09-21T10:03:19.765036+0000","last_became_peered":"2024-09-21T10:03:19.765036+0000","last_unstale":"2024-09-21T10:07:09.517045+0000","last_undegraded":"2024-09-21T10:07:09.517045+0000","last_fullsized":"2024-09-21T10:07:09.517045+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:32.230377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":129,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.521405+0000","last_change":"2024-09-21T10:02:46.660814+0000","last_active":"2024-09-21T10:07:09.521405+0000","last_peered":"2024-09-21T10:07:09.521405+0000","last_clean":"2024-09-21T10:07:09.521405+0000","last_became_active":"2024-09-21T10:02:46.660325+0000","last_became_peered":"2024-09-21T10:02:46.660325+0000","last_unstale":"2024-09-21T10:07:09.521405+0000","last_undegraded":"2024-09-21T10:07:09.521405+0000","last_fullsized":"2024-09-21T10:07:09.521405+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:44.427243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"129'9543","reported_seq":9587,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.084541+0000","last_change":"2024-09-21T10:05:19.213938+0000","last_active":"2024-09-21T10:07:21.084541+0000","last_peered":"2024-09-21T10:07:21.084541+0000","last_clean":"2024-09-21T10:07:21.084541+0000","last_became_active":"2024-09-21T10:05:19.213559+0000","last_became_peered":"2024-09-21T10:05:19.213559+0000","last_unstale":"2024-09-21T10:07:21.084541+0000","last_undegraded":"2024-09-21T10:07:21.084541+0000","last_fullsized":"2024-09-21T10:07:21.084541+0000","mapping_epoch":112,"log_start":"117'700","ondisk_log_start":"117'700","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8843,"log_dups_size":0,"ondisk_log_size":8843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:25.094755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":465764352,"num_objects":7107,"num_object_clones":0,"num_object_copies":21321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9543,"num_write_kb":532800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":831,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.328568+0000","last_change":"2024-09-21T10:02:46.657107+0000","last_active":"2024-09-21T10:06:58.328568+0000","last_peered":"2024-09-21T10:06:58.328568+0000","last_clean":"2024-09-21T10:06:58.328568+0000","last_became_active":"2024-09-21T10:02:46.656633+0000","last_became_peered":"2024-09-21T10:02:46.656633+0000","last_unstale":"2024-09-21T10:06:58.328568+0000","last_undegraded":"2024-09-21T10:06:58.328568+0000","last_fullsized":"2024-09-21T10:06:58.328568+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:42:18.534832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:07:04.169226+0000","last_change":"2024-09-21T10:02:12.840581+0000","last_active":"2024-09-21T10:07:04.169226+0000","last_peered":"2024-09-21T10:07:04.169226+0000","last_clean":"2024-09-21T10:07:04.169226+0000","last_became_active":"2024-09-21T10:02:12.840033+0000","last_became_peered":"2024-09-21T10:02:12.840033+0000","last_unstale":"2024-09-21T10:07:04.169226+0000","last_undegraded":"2024-09-21T10:07:04.169226+0000","last_fullsized":"2024-09-21T10:07:04.169226+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:22.449586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":107,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.517974+0000","last_change":"2024-09-21T10:03:53.843551+0000","last_active":"2024-09-21T10:07:09.517974+0000","last_peered":"2024-09-21T10:07:09.517974+0000","last_clean":"2024-09-21T10:07:09.517974+0000","last_became_active":"2024-09-21T10:03:53.843007+0000","last_became_peered":"2024-09-21T10:03:53.843007+0000","last_unstale":"2024-09-21T10:07:09.517974+0000","last_undegraded":"2024-09-21T10:07:09.517974+0000","last_fullsized":"2024-09-21T10:07:09.517974+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:11.757447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"129'9546","reported_seq":9591,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.018357+0000","last_change":"2024-09-21T10:05:19.212703+0000","last_active":"2024-09-21T10:07:21.018357+0000","last_peered":"2024-09-21T10:07:21.018357+0000","last_clean":"2024-09-21T10:07:21.018357+0000","last_became_active":"2024-09-21T10:05:19.212420+0000","last_became_peered":"2024-09-21T10:05:19.212420+0000","last_unstale":"2024-09-21T10:07:21.018357+0000","last_undegraded":"2024-09-21T10:07:21.018357+0000","last_fullsized":"2024-09-21T10:07:21.018357+0000","mapping_epoch":112,"log_start":"116'700","ondisk_log_start":"116'700","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8846,"log_dups_size":0,"ondisk_log_size":8846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:43.062243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":466485248,"num_objects":7118,"num_object_clones":0,"num_object_copies":21354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9546,"num_write_kb":533248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.258865+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:07:09.258865+0000","last_peered":"2024-09-21T10:07:09.258865+0000","last_clean":"2024-09-21T10:07:09.258865+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:07:09.258865+0000","last_undegraded":"2024-09-21T10:07:09.258865+0000","last_fullsized":"2024-09-21T10:07:09.258865+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":569,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.518065+0000","last_change":"2024-09-21T10:02:46.662410+0000","last_active":"2024-09-21T10:07:09.518065+0000","last_peered":"2024-09-21T10:07:09.518065+0000","last_clean":"2024-09-21T10:07:09.518065+0000","last_became_active":"2024-09-21T10:02:46.661506+0000","last_became_peered":"2024-09-21T10:02:46.661506+0000","last_unstale":"2024-09-21T10:07:09.518065+0000","last_undegraded":"2024-09-21T10:07:09.518065+0000","last_fullsized":"2024-09-21T10:07:09.518065+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:40:33.566844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":85,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.517411+0000","last_change":"2024-09-21T10:02:46.665713+0000","last_active":"2024-09-21T10:07:09.517411+0000","last_peered":"2024-09-21T10:07:09.517411+0000","last_clean":"2024-09-21T10:07:09.517411+0000","last_became_active":"2024-09-21T10:02:46.665316+0000","last_became_peered":"2024-09-21T10:02:46.665316+0000","last_unstale":"2024-09-21T10:07:09.517411+0000","last_undegraded":"2024-09-21T10:07:09.517411+0000","last_fullsized":"2024-09-21T10:07:09.517411+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.951202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"129'9407","reported_seq":9431,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:19.143760+0000","last_change":"2024-09-21T10:05:19.214304+0000","last_active":"2024-09-21T10:07:19.143760+0000","last_peered":"2024-09-21T10:07:19.143760+0000","last_clean":"2024-09-21T10:07:19.143760+0000","last_became_active":"2024-09-21T10:05:19.213946+0000","last_became_peered":"2024-09-21T10:05:19.213946+0000","last_unstale":"2024-09-21T10:07:19.143760+0000","last_undegraded":"2024-09-21T10:07:19.143760+0000","last_fullsized":"2024-09-21T10:07:19.143760+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9407,"log_dups_size":0,"ondisk_log_size":9407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:18.695450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":462618624,"num_objects":7059,"num_object_clones":0,"num_object_copies":21177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9407,"num_write_kb":526912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":94,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.518276+0000","last_change":"2024-09-21T10:03:19.768187+0000","last_active":"2024-09-21T10:07:09.518276+0000","last_peered":"2024-09-21T10:07:09.518276+0000","last_clean":"2024-09-21T10:07:09.518276+0000","last_became_active":"2024-09-21T10:03:19.767801+0000","last_became_peered":"2024-09-21T10:03:19.767801+0000","last_unstale":"2024-09-21T10:07:09.518276+0000","last_undegraded":"2024-09-21T10:07:09.518276+0000","last_fullsized":"2024-09-21T10:07:09.518276+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:38.042481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":531,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.325900+0000","last_change":"2024-09-21T10:02:12.840296+0000","last_active":"2024-09-21T10:06:58.325900+0000","last_peered":"2024-09-21T10:06:58.325900+0000","last_clean":"2024-09-21T10:06:58.325900+0000","last_became_active":"2024-09-21T10:02:12.839881+0000","last_became_peered":"2024-09-21T10:02:12.839881+0000","last_unstale":"2024-09-21T10:06:58.325900+0000","last_undegraded":"2024-09-21T10:06:58.325900+0000","last_fullsized":"2024-09-21T10:06:58.325900+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:49.166865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":95,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.521423+0000","last_change":"2024-09-21T10:02:12.845158+0000","last_active":"2024-09-21T10:07:09.521423+0000","last_peered":"2024-09-21T10:07:09.521423+0000","last_clean":"2024-09-21T10:07:09.521423+0000","last_became_active":"2024-09-21T10:02:12.843399+0000","last_became_peered":"2024-09-21T10:02:12.843399+0000","last_unstale":"2024-09-21T10:07:09.521423+0000","last_undegraded":"2024-09-21T10:07:09.521423+0000","last_fullsized":"2024-09-21T10:07:09.521423+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:27.552491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"129'9270","reported_seq":9296,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:19.154195+0000","last_change":"2024-09-21T10:05:19.214110+0000","last_active":"2024-09-21T10:07:19.154195+0000","last_peered":"2024-09-21T10:07:19.154195+0000","last_clean":"2024-09-21T10:07:19.154195+0000","last_became_active":"2024-09-21T10:05:19.213716+0000","last_became_peered":"2024-09-21T10:05:19.213716+0000","last_unstale":"2024-09-21T10:07:19.154195+0000","last_undegraded":"2024-09-21T10:07:19.154195+0000","last_fullsized":"2024-09-21T10:07:19.154195+0000","mapping_epoch":112,"log_start":"117'1900","ondisk_log_start":"117'1900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":7370,"log_dups_size":0,"ondisk_log_size":7370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:20.723064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":473956352,"num_objects":7232,"num_object_clones":0,"num_object_copies":21696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9270,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":112,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.505078+0000","last_change":"2024-09-21T10:03:53.842610+0000","last_active":"2024-09-21T10:07:09.505078+0000","last_peered":"2024-09-21T10:07:09.505078+0000","last_clean":"2024-09-21T10:07:09.505078+0000","last_became_active":"2024-09-21T10:03:53.841888+0000","last_became_peered":"2024-09-21T10:03:53.841888+0000","last_unstale":"2024-09-21T10:07:09.505078+0000","last_undegraded":"2024-09-21T10:07:09.505078+0000","last_fullsized":"2024-09-21T10:07:09.505078+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:05.833771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":675,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.518133+0000","last_change":"2024-09-21T10:02:46.665711+0000","last_active":"2024-09-21T10:07:09.518133+0000","last_peered":"2024-09-21T10:07:09.518133+0000","last_clean":"2024-09-21T10:07:09.518133+0000","last_became_active":"2024-09-21T10:02:46.665165+0000","last_became_peered":"2024-09-21T10:02:46.665165+0000","last_unstale":"2024-09-21T10:07:09.518133+0000","last_undegraded":"2024-09-21T10:07:09.518133+0000","last_fullsized":"2024-09-21T10:07:09.518133+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:08.886554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":109,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.506500+0000","last_change":"2024-09-21T10:03:53.838941+0000","last_active":"2024-09-21T10:07:09.506500+0000","last_peered":"2024-09-21T10:07:09.506500+0000","last_clean":"2024-09-21T10:07:09.506500+0000","last_became_active":"2024-09-21T10:03:53.838646+0000","last_became_peered":"2024-09-21T10:03:53.838646+0000","last_unstale":"2024-09-21T10:07:09.506500+0000","last_undegraded":"2024-09-21T10:07:09.506500+0000","last_fullsized":"2024-09-21T10:07:09.506500+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:25.167057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"129'9610","reported_seq":9655,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.677044+0000","last_change":"2024-09-21T10:05:19.210763+0000","last_active":"2024-09-21T10:07:21.677044+0000","last_peered":"2024-09-21T10:07:21.677044+0000","last_clean":"2024-09-21T10:07:21.677044+0000","last_became_active":"2024-09-21T10:05:19.210462+0000","last_became_peered":"2024-09-21T10:05:19.210462+0000","last_unstale":"2024-09-21T10:07:21.677044+0000","last_undegraded":"2024-09-21T10:07:21.677044+0000","last_fullsized":"2024-09-21T10:07:21.677044+0000","mapping_epoch":112,"log_start":"116'700","ondisk_log_start":"116'700","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8910,"log_dups_size":0,"ondisk_log_size":8910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:57.912861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":463994880,"num_objects":7080,"num_object_clones":0,"num_object_copies":21240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9610,"num_write_kb":534080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":109,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.519674+0000","last_change":"2024-09-21T10:02:46.661363+0000","last_active":"2024-09-21T10:07:09.519674+0000","last_peered":"2024-09-21T10:07:09.519674+0000","last_clean":"2024-09-21T10:07:09.519674+0000","last_became_active":"2024-09-21T10:02:46.661005+0000","last_became_peered":"2024-09-21T10:02:46.661005+0000","last_unstale":"2024-09-21T10:07:09.519674+0000","last_undegraded":"2024-09-21T10:07:09.519674+0000","last_fullsized":"2024-09-21T10:07:09.519674+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:42.908471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":369,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.320802+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:06:58.320802+0000","last_peered":"2024-09-21T10:06:58.320802+0000","last_clean":"2024-09-21T10:06:58.320802+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:06:58.320802+0000","last_undegraded":"2024-09-21T10:06:58.320802+0000","last_fullsized":"2024-09-21T10:06:58.320802+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:59.141918+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:06:59.141918+0000","last_peered":"2024-09-21T10:06:59.141918+0000","last_clean":"2024-09-21T10:06:59.141918+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:06:59.141918+0000","last_undegraded":"2024-09-21T10:06:59.141918+0000","last_fullsized":"2024-09-21T10:06:59.141918+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":92,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.326347+0000","last_change":"2024-09-21T10:02:46.667831+0000","last_active":"2024-09-21T10:06:58.326347+0000","last_peered":"2024-09-21T10:06:58.326347+0000","last_clean":"2024-09-21T10:06:58.326347+0000","last_became_active":"2024-09-21T10:02:46.667507+0000","last_became_peered":"2024-09-21T10:02:46.667507+0000","last_unstale":"2024-09-21T10:06:58.326347+0000","last_undegraded":"2024-09-21T10:06:58.326347+0000","last_fullsized":"2024-09-21T10:06:58.326347+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:23.990802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":101,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.328867+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:06:58.328867+0000","last_peered":"2024-09-21T10:06:58.328867+0000","last_clean":"2024-09-21T10:06:58.328867+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:06:58.328867+0000","last_undegraded":"2024-09-21T10:06:58.328867+0000","last_fullsized":"2024-09-21T10:06:58.328867+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.328367+0000","last_change":"2024-09-21T10:03:19.767147+0000","last_active":"2024-09-21T10:06:58.328367+0000","last_peered":"2024-09-21T10:06:58.328367+0000","last_clean":"2024-09-21T10:06:58.328367+0000","last_became_active":"2024-09-21T10:03:19.766822+0000","last_became_peered":"2024-09-21T10:03:19.766822+0000","last_unstale":"2024-09-21T10:06:58.328367+0000","last_undegraded":"2024-09-21T10:06:58.328367+0000","last_fullsized":"2024-09-21T10:06:58.328367+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:38.875912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":96,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.514491+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:07:09.514491+0000","last_peered":"2024-09-21T10:07:09.514491+0000","last_clean":"2024-09-21T10:07:09.514491+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:07:09.514491+0000","last_undegraded":"2024-09-21T10:07:09.514491+0000","last_fullsized":"2024-09-21T10:07:09.514491+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.511800+0000","last_change":"2024-09-21T10:02:46.659383+0000","last_active":"2024-09-21T10:07:09.511800+0000","last_peered":"2024-09-21T10:07:09.511800+0000","last_clean":"2024-09-21T10:07:09.511800+0000","last_became_active":"2024-09-21T10:02:46.659011+0000","last_became_peered":"2024-09-21T10:02:46.659011+0000","last_unstale":"2024-09-21T10:07:09.511800+0000","last_undegraded":"2024-09-21T10:07:09.511800+0000","last_fullsized":"2024-09-21T10:07:09.511800+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:04.650699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.507143+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:07:09.507143+0000","last_peered":"2024-09-21T10:07:09.507143+0000","last_clean":"2024-09-21T10:07:09.507143+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:07:09.507143+0000","last_undegraded":"2024-09-21T10:07:09.507143+0000","last_fullsized":"2024-09-21T10:07:09.507143+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":104,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.520145+0000","last_change":"2024-09-21T10:03:53.844673+0000","last_active":"2024-09-21T10:07:09.520145+0000","last_peered":"2024-09-21T10:07:09.520145+0000","last_clean":"2024-09-21T10:07:09.520145+0000","last_became_active":"2024-09-21T10:03:53.844313+0000","last_became_peered":"2024-09-21T10:03:53.844313+0000","last_unstale":"2024-09-21T10:07:09.520145+0000","last_undegraded":"2024-09-21T10:07:09.520145+0000","last_fullsized":"2024-09-21T10:07:09.520145+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:42.529301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":127,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.506357+0000","last_change":"2024-09-21T10:03:53.841417+0000","last_active":"2024-09-21T10:07:09.506357+0000","last_peered":"2024-09-21T10:07:09.506357+0000","last_clean":"2024-09-21T10:07:09.506357+0000","last_became_active":"2024-09-21T10:03:53.840993+0000","last_became_peered":"2024-09-21T10:03:53.840993+0000","last_unstale":"2024-09-21T10:07:09.506357+0000","last_undegraded":"2024-09-21T10:07:09.506357+0000","last_fullsized":"2024-09-21T10:07:09.506357+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:05.666490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:59.121266+0000","last_change":"2024-09-21T10:03:19.765682+0000","last_active":"2024-09-21T10:06:59.121266+0000","last_peered":"2024-09-21T10:06:59.121266+0000","last_clean":"2024-09-21T10:06:59.121266+0000","last_became_active":"2024-09-21T10:03:19.765344+0000","last_became_peered":"2024-09-21T10:03:19.765344+0000","last_unstale":"2024-09-21T10:06:59.121266+0000","last_undegraded":"2024-09-21T10:06:59.121266+0000","last_fullsized":"2024-09-21T10:06:59.121266+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:43.787280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":130,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.506509+0000","last_change":"2024-09-21T10:03:53.839372+0000","last_active":"2024-09-21T10:07:09.506509+0000","last_peered":"2024-09-21T10:07:09.506509+0000","last_clean":"2024-09-21T10:07:09.506509+0000","last_became_active":"2024-09-21T10:03:53.839053+0000","last_became_peered":"2024-09-21T10:03:53.839053+0000","last_unstale":"2024-09-21T10:07:09.506509+0000","last_undegraded":"2024-09-21T10:07:09.506509+0000","last_fullsized":"2024-09-21T10:07:09.506509+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:29.647982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":106,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.522484+0000","last_change":"2024-09-21T10:03:53.844946+0000","last_active":"2024-09-21T10:07:09.522484+0000","last_peered":"2024-09-21T10:07:09.522484+0000","last_clean":"2024-09-21T10:07:09.522484+0000","last_became_active":"2024-09-21T10:03:53.844613+0000","last_became_peered":"2024-09-21T10:03:53.844613+0000","last_unstale":"2024-09-21T10:07:09.522484+0000","last_undegraded":"2024-09-21T10:07:09.522484+0000","last_fullsized":"2024-09-21T10:07:09.522484+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:41.055573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.517675+0000","last_change":"2024-09-21T10:03:53.846948+0000","last_active":"2024-09-21T10:07:09.517675+0000","last_peered":"2024-09-21T10:07:09.517675+0000","last_clean":"2024-09-21T10:07:09.517675+0000","last_became_active":"2024-09-21T10:03:53.846614+0000","last_became_peered":"2024-09-21T10:03:53.846614+0000","last_unstale":"2024-09-21T10:07:09.517675+0000","last_undegraded":"2024-09-21T10:07:09.517675+0000","last_fullsized":"2024-09-21T10:07:09.517675+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:53.968372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":113,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504455+0000","last_change":"2024-09-21T10:03:53.845201+0000","last_active":"2024-09-21T10:07:09.504455+0000","last_peered":"2024-09-21T10:07:09.504455+0000","last_clean":"2024-09-21T10:07:09.504455+0000","last_became_active":"2024-09-21T10:03:53.843822+0000","last_became_peered":"2024-09-21T10:03:53.843822+0000","last_unstale":"2024-09-21T10:07:09.504455+0000","last_undegraded":"2024-09-21T10:07:09.504455+0000","last_fullsized":"2024-09-21T10:07:09.504455+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:28.895373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":108,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.329205+0000","last_change":"2024-09-21T10:03:19.768025+0000","last_active":"2024-09-21T10:06:58.329205+0000","last_peered":"2024-09-21T10:06:58.329205+0000","last_clean":"2024-09-21T10:06:58.329205+0000","last_became_active":"2024-09-21T10:03:19.767632+0000","last_became_peered":"2024-09-21T10:03:19.767632+0000","last_unstale":"2024-09-21T10:06:58.329205+0000","last_undegraded":"2024-09-21T10:06:58.329205+0000","last_fullsized":"2024-09-21T10:06:58.329205+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:40.023759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":133,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.516825+0000","last_change":"2024-09-21T10:03:53.840263+0000","last_active":"2024-09-21T10:07:09.516825+0000","last_peered":"2024-09-21T10:07:09.516825+0000","last_clean":"2024-09-21T10:07:09.516825+0000","last_became_active":"2024-09-21T10:03:53.839894+0000","last_became_peered":"2024-09-21T10:03:53.839894+0000","last_unstale":"2024-09-21T10:07:09.516825+0000","last_undegraded":"2024-09-21T10:07:09.516825+0000","last_fullsized":"2024-09-21T10:07:09.516825+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:52.489580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":98,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.529933+0000","last_change":"2024-09-21T10:02:12.846054+0000","last_active":"2024-09-21T10:07:09.529933+0000","last_peered":"2024-09-21T10:07:09.529933+0000","last_clean":"2024-09-21T10:07:09.529933+0000","last_became_active":"2024-09-21T10:02:12.843696+0000","last_became_peered":"2024-09-21T10:02:12.843696+0000","last_unstale":"2024-09-21T10:07:09.529933+0000","last_undegraded":"2024-09-21T10:07:09.529933+0000","last_fullsized":"2024-09-21T10:07:09.529933+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:32.280697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":106,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.326271+0000","last_change":"2024-09-21T10:03:53.840992+0000","last_active":"2024-09-21T10:06:58.326271+0000","last_peered":"2024-09-21T10:06:58.326271+0000","last_clean":"2024-09-21T10:06:58.326271+0000","last_became_active":"2024-09-21T10:03:53.840551+0000","last_became_peered":"2024-09-21T10:03:53.840551+0000","last_unstale":"2024-09-21T10:06:58.326271+0000","last_undegraded":"2024-09-21T10:06:58.326271+0000","last_fullsized":"2024-09-21T10:06:58.326271+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:04.442126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.323840+0000","last_change":"2024-09-21T10:02:46.659513+0000","last_active":"2024-09-21T10:06:58.323840+0000","last_peered":"2024-09-21T10:06:58.323840+0000","last_clean":"2024-09-21T10:06:58.323840+0000","last_became_active":"2024-09-21T10:02:46.659130+0000","last_became_peered":"2024-09-21T10:02:46.659130+0000","last_unstale":"2024-09-21T10:06:58.323840+0000","last_undegraded":"2024-09-21T10:06:58.323840+0000","last_fullsized":"2024-09-21T10:06:58.323840+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:24.792175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":106,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.329088+0000","last_change":"2024-09-21T10:03:53.835006+0000","last_active":"2024-09-21T10:06:58.329088+0000","last_peered":"2024-09-21T10:06:58.329088+0000","last_clean":"2024-09-21T10:06:58.329088+0000","last_became_active":"2024-09-21T10:03:53.834608+0000","last_became_peered":"2024-09-21T10:03:53.834608+0000","last_unstale":"2024-09-21T10:06:58.329088+0000","last_undegraded":"2024-09-21T10:06:58.329088+0000","last_fullsized":"2024-09-21T10:06:58.329088+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:56.108606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":79,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-09-21T10:06:58.324220+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:06:58.324220+0000","last_peered":"2024-09-21T10:06:58.324220+0000","last_clean":"2024-09-21T10:06:58.324220+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:06:58.324220+0000","last_undegraded":"2024-09-21T10:06:58.324220+0000","last_fullsized":"2024-09-21T10:06:58.324220+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.512496+0000","last_change":"2024-09-21T10:03:53.844043+0000","last_active":"2024-09-21T10:07:09.512496+0000","last_peered":"2024-09-21T10:07:09.512496+0000","last_clean":"2024-09-21T10:07:09.512496+0000","last_became_active":"2024-09-21T10:03:53.842891+0000","last_became_peered":"2024-09-21T10:03:53.842891+0000","last_unstale":"2024-09-21T10:07:09.512496+0000","last_undegraded":"2024-09-21T10:07:09.512496+0000","last_fullsized":"2024-09-21T10:07:09.512496+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:39.891977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":121,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504805+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:07:09.504805+0000","last_peered":"2024-09-21T10:07:09.504805+0000","last_clean":"2024-09-21T10:07:09.504805+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:07:09.504805+0000","last_undegraded":"2024-09-21T10:07:09.504805+0000","last_fullsized":"2024-09-21T10:07:09.504805+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":92,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.505133+0000","last_change":"2024-09-21T10:02:46.659818+0000","last_active":"2024-09-21T10:07:09.505133+0000","last_peered":"2024-09-21T10:07:09.505133+0000","last_clean":"2024-09-21T10:07:09.505133+0000","last_became_active":"2024-09-21T10:02:46.659457+0000","last_became_peered":"2024-09-21T10:02:46.659457+0000","last_unstale":"2024-09-21T10:07:09.505133+0000","last_undegraded":"2024-09-21T10:07:09.505133+0000","last_fullsized":"2024-09-21T10:07:09.505133+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:21.962330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":109,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.519801+0000","last_change":"2024-09-21T10:03:19.769549+0000","last_active":"2024-09-21T10:07:09.519801+0000","last_peered":"2024-09-21T10:07:09.519801+0000","last_clean":"2024-09-21T10:07:09.519801+0000","last_became_active":"2024-09-21T10:03:19.768854+0000","last_became_peered":"2024-09-21T10:03:19.768854+0000","last_unstale":"2024-09-21T10:07:09.519801+0000","last_undegraded":"2024-09-21T10:07:09.519801+0000","last_fullsized":"2024-09-21T10:07:09.519801+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:29.415039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":7572946968,"num_objects":115555,"num_object_clones":0,"num_object_copies":346665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":115555,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":151223,"num_write_kb":8536833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":22661181440,"data_stored":22661169224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142223,"ondisk_log_size":142223,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234212,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2410924,"kb_used_data":2335524,"kb_used_omap":1,"kb_used_meta":75390,"kb_avail":91334228,"statfs":{"total":95995035648,"available":93526249472,"internally_reserved":0,"allocated":2391576576,"data_stored":2390888102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":77199819},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495850,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1926180,"kb_used_data":1850780,"kb_used_omap":1,"kb_used_meta":75390,"kb_avail":91818972,"statfs":{"total":95995035648,"available":94022627328,"internally_reserved":0,"allocated":1895198720,"data_stored":1894518618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":77199815},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757487,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4293212,"kb_used_data":4167764,"kb_used_omap":1,"kb_used_meta":125438,"kb_avail":89451940,"statfs":{"total":95995035648,"available":91598786560,"internally_reserved":0,"allocated":4267790336,"data_stored":4267113732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":128448966},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019123,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3815524,"kb_used_data":3712220,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":89929628,"statfs":{"total":95995035648,"available":92087939072,"internally_reserved":0,"allocated":3801313280,"data_stored":3800632691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":105773510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435308,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1417680,"kb_used_data":1372104,"kb_used_omap":5,"kb_used_meta":45562,"kb_avail":92327472,"statfs":{"total":95995035648,"available":94543331328,"internally_reserved":0,"allocated":1405034496,"data_stored":1404369587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":46655528},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598834,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2900500,"kb_used_data":2814668,"kb_used_omap":7,"kb_used_meta":85816,"kb_avail":90844652,"statfs":{"total":95995035648,"available":93024923648,"internally_reserved":0,"allocated":2882220032,"data_stored":2881551269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":87876045},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762361,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2406028,"kb_used_data":2329860,"kb_used_omap":7,"kb_used_meta":76088,"kb_avail":91339124,"statfs":{"total":95995035648,"available":93531262976,"internally_reserved":0,"allocated":2385776640,"data_stored":2385115483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":77914365},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925888,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3781272,"kb_used_data":3669072,"kb_used_omap":7,"kb_used_meta":112184,"kb_avail":89963880,"statfs":{"total":95995035648,"available":92123013120,"internally_reserved":0,"allocated":3757129728,"data_stored":3756457685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":114876606},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3722510336,"data_stored":3722510336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2364932096,"data_stored":2364932096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2868121600,"data_stored":2868117528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1398407168,"data_stored":1398407168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3797352448,"data_stored":3797352448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4253683712,"data_stored":4253679640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1885798400,"data_stored":1885798400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2370375680,"data_stored":2370371608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:07:24.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:07:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:24.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:24.651 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:24.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.704 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:24.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:24.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:24 smithi032 ceph-mon[105748]: from='client.34731 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:24.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:24 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:24.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:24 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1262764286' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:24.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:24 smithi032 ceph-mon[105748]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T10:07:24.738 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:24 smithi032 ceph-mon[101299]: from='client.34731 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:24.738 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:24 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:24.738 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:24 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1262764286' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:24.738 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:24 smithi032 ceph-mon[101299]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T10:07:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:24 smithi045 ceph-mon[95976]: from='client.34731 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:24 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:24 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1262764286' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:24 smithi045 ceph-mon[95976]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T10:07:24.804 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:24.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:24.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:24.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:24.982 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:24.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:25.083 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:25.112 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:25.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:25.246 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:25.347 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:25.390 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:25.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:25.492 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:25.526 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:25.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:25.628 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:25.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:25.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:25.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:25 smithi032 ceph-mon[105748]: pgmap v376: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-09-21T10:07:25.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:25 smithi032 ceph-mon[101299]: pgmap v376: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:26445 5.0 5:00724b50:::benchmark_data_smithi032_140417_object26444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:38.493544+0000", 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 106.810382371, 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16464266, 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.739 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26445 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493544+0000", 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.740 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493544+0000", 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493546+0000", 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.917e-06 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493574+0000", 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.7492999999999999e-05 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493602+0000", 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.8076999999999999e-05 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493612+0000", 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0227999999999999e-05 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.493869+0000", 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00025694200000000002 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.494037+0000", 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016832199999999999 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.494265+0000", 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00022723 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495580+0000", 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001315379 2024-09-21T10:07:25.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.497352+0000", 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001771667 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.658107+0000", 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16075497999999999 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.658151+0000", 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4885999999999997e-05 2024-09-21T10:07:25.744 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.658187+0000", 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5539e-05 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:26447 5.0 5:0e6e8a43:::benchmark_data_smithi032_140417_object26446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:38.495608+0000", 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 106.808318661, 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16463739299999999, 2024-09-21T10:07:25.745 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26447 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495608+0000", 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.746 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495608+0000", 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495609+0000", 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.344e-06 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495635+0000", 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5448000000000001e-05 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.747 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495648+0000", 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2860999999999999e-05 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495656+0000", 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2830000000000001e-06 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495739+0000", 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3478000000000003e-05 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.748 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.495881+0000", 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014165799999999999 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496033+0000", 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015200200000000001 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.497367+0000", 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0013334130000000001 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.749 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.498680+0000", 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00131339 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.660077+0000", 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16139730299999999 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.660206+0000", 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012886700000000001 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.750 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.660245+0000", 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9345999999999999e-05 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:34930 5.0 5:0b1d4d0a:::benchmark_data_smithi032_140417_object34929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:42.475909+0000", 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 102.828017436, 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.073159774999999996, 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.751 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 34930 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.475909+0000", 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.475909+0000", 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.752 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.475911+0000", 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.72e-06 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.475936+0000", 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4624999999999999e-05 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.475949+0000", 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.307e-05 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.753 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.476060+0000", 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011084800000000001 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.476911+0000", 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00085193599999999997 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.477031+0000", 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000119515 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.754 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.477164+0000", 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000132669 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.479289+0000", 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0021249709999999998 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.480405+0000", 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001116258 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.755 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.548912+0000", 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.068506681 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.549013+0000", 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000100931 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.549069+0000", 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6551000000000001e-05 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.756 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:34958 5.0 5:08d20ca9:::benchmark_data_smithi032_140417_object34957:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:42.486743+0000", 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 102.81718325, 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.062475847000000001, 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 34958 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.757 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.486743+0000", 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.486743+0000", 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.486745+0000", 2024-09-21T10:07:25.758 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9e-06 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.486782+0000", 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6400999999999999e-05 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.486795+0000", 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.367e-05 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.759 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.486803+0000", 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0139999999999998e-06 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.486898+0000", 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4067999999999995e-05 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.487026+0000", 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012831799999999999 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.760 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.487130+0000", 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010366600000000001 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.488275+0000", 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0011456050000000001 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.489006+0000", 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00073107700000000001 2024-09-21T10:07:25.761 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.549129+0000", 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.060123283999999999 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.549183+0000", 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3662999999999998e-05 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:42.549219+0000", 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6180999999999998e-05 2024-09-21T10:07:25.762 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:37738 5.0 5:04f0838a:::benchmark_data_smithi032_140417_object37737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:43.960332+0000", 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 101.343594962, 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.058513998999999997, 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 37738 2024-09-21T10:07:25.763 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960332+0000", 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960332+0000", 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960333+0000", 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0499999999999999e-06 2024-09-21T10:07:25.764 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960361+0000", 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.8116999999999999e-05 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960372+0000", 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1113e-05 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960379+0000", 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7229999999999998e-06 2024-09-21T10:07:25.765 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960435+0000", 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6542000000000002e-05 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960554+0000", 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011833900000000001 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.960688+0000", 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013446700000000001 2024-09-21T10:07:25.766 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.961915+0000", 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001226937 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:43.962825+0000", 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00090990299999999999 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018741+0000", 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.055915858999999998 2024-09-21T10:07:25.767 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018802+0000", 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0711999999999997e-05 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:44.018846+0000", 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4236999999999997e-05 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.768 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:40113 5.0 5:065f77a4:::benchmark_data_smithi032_140417_object40112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:45.332635+0000", 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 99.971291944000001, 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.059432579999999999, 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 40113 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.769 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332635+0000", 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332635+0000", 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.770 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332636+0000", 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.486e-06 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332659+0000", 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2841999999999999e-05 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332671+0000", 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2234e-05 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.771 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332680+0000", 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2519999999999999e-06 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332732+0000", 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2012000000000003e-05 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332818+0000", 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6439000000000004e-05 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.772 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.332920+0000", 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000101508 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.333937+0000", 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001017888 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.334994+0000", 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0010565520000000001 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.773 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.391964+0000", 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.056970135999999998 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.392023+0000", 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8940000000000002e-05 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.392067+0000", 2024-09-21T10:07:25.774 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4291e-05 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:40118 5.0 5:097060d4:::benchmark_data_smithi032_140417_object40117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:45.335520+0000", 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 99.968406525999995, 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.056731358000000003, 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.775 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 40118 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.335520+0000", 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.335520+0000", 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.776 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.335521+0000", 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.274e-06 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.335542+0000", 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0157e-05 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.335554+0000", 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2082999999999999e-05 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.777 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.335562+0000", 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8290000000000008e-06 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.335618+0000", 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6436e-05 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.335745+0000", 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012681899999999999 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.778 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.335921+0000", 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017570699999999999 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.337150+0000", 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001229929 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.337850+0000", 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00069999299999999999 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.779 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.392137+0000", 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.054286134 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.392204+0000", 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7457000000000001e-05 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.392252+0000", 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7540000000000002e-05 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.780 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:40119 5.1 e117/112 5:8ab9f3b6:::benchmark_data_smithi032_140417_object40118:head v 117'2410, mlcod=117'2408)", 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:45.336984+0000", 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 99.966942974000006, 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.050071147000000003, 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.336984+0000", 2024-09-21T10:07:25.781 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.336984+0000", 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.336984+0000", 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3900000000000004e-07 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.337055+0000", 2024-09-21T10:07:25.782 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0851000000000003e-05 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.337064+0000", 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4400000000000005e-06 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.337068+0000", 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4120000000000003e-06 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.337113+0000", 2024-09-21T10:07:25.783 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.515e-05 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.337132+0000", 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8964e-05 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.338232+0000", 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001099404 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.784 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:45.387055+0000", 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.048823186999999997 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:87051 5.0 5:0acf7c1d:::benchmark_data_smithi032_140417_object87050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:15.704361+0000", 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 69.599565487999996, 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.039936015999999998, 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.785 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 87051 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.704361+0000", 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.704361+0000", 2024-09-21T10:07:25.786 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.704362+0000", 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2190000000000001e-06 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.704381+0000", 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8397999999999999e-05 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.704390+0000", 2024-09-21T10:07:25.787 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0170000000000002e-06 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.704395+0000", 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1739999999999999e-06 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.704437+0000", 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2349999999999999e-05 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.704532+0000", 2024-09-21T10:07:25.788 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4987000000000004e-05 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.704621+0000", 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8630999999999999e-05 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.707129+0000", 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0025081109999999999 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708209+0000", 2024-09-21T10:07:25.789 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001079974 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.743987+0000", 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.035777528000000003 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.744164+0000", 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017722899999999999 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.790 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.744297+0000", 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013339800000000001 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:87056 5.0 5:0fcfd5af:::benchmark_data_smithi032_140417_object87055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:15.708144+0000", 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 69.595783033999993, 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.041243521999999998, 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.791 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 87056 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708144+0000", 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708144+0000", 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.792 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708145+0000", 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.514e-06 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708181+0000", 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5738000000000002e-05 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708192+0000", 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0634e-05 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.793 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708197+0000", 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5130000000000004e-06 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708288+0000", 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0890000000000006e-05 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708439+0000", 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015079700000000001 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.794 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.708586+0000", 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014715 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.711182+0000", 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0025958349999999999 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.711331+0000", 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000149035 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.795 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.748803+0000", 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.037472008000000001 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.749197+0000", 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00039414000000000002 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:15.749387+0000", 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000190268 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.796 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:88558 5.0 5:0f02bdf3:::benchmark_data_smithi032_140417_object88557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:17.007874+0000", 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 68.296052631999999, 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.075857958000000003, 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 88558 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.797 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.007874+0000", 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.007874+0000", 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.007875+0000", 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.263e-06 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.798 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.007892+0000", 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6540000000000001e-05 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.007902+0000", 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0222e-05 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.007908+0000", 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.468e-06 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.799 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.007950+0000", 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2552999999999997e-05 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.008037+0000", 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7114999999999995e-05 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.008134+0000", 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6337999999999998e-05 2024-09-21T10:07:25.800 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.010278+0000", 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00214437 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.010825+0000", 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00054701000000000001 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013463+0000", 2024-09-21T10:07:25.801 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002637679 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013496+0000", 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2956000000000001e-05 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.083732+0000", 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.070236443999999995 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.802 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44543.0:88558 5.0 e122/112 ondisk, result = 0)", 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:17.013396+0000", 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 68.290531099999995, 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.070345863999999994, 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013396+0000", 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.803 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013396+0000", 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013396+0000", 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.61e-07 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013400+0000", 2024-09-21T10:07:25.804 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4230000000000001e-06 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013407+0000", 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4050000000000003e-06 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013411+0000", 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0980000000000004e-06 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.805 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013455+0000", 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3757999999999997e-05 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.013460+0000", 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6829999999999999e-06 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.083741+0000", 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.070281836 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.806 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:88569 5.1 e122/112 5:883b3bda:::benchmark_data_smithi032_140417_object88568:head v 122'5415, mlcod=122'5413)", 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:17.016063+0000", 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 68.287863740999995, 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.070082940999999996, 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.016063+0000", 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.807 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.016063+0000", 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.016064+0000", 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0699999999999997e-07 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.016129+0000", 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5267999999999999e-05 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.808 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.082792+0000", 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.066662885000000005 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.082804+0000", 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2094e-05 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.082904+0000", 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000100444 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.809 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.082932+0000", 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.7597e-05 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.086094+0000", 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003162434 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:17.086146+0000", 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1612e-05 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.810 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90934 5.0 5:09153f62:::benchmark_data_smithi032_140417_object90933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:19.273026+0000", 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 66.030900247000005, 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.065652258000000005, 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90934 2024-09-21T10:07:25.811 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.273026+0000", 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.273026+0000", 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.273028+0000", 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.155e-06 2024-09-21T10:07:25.812 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.273046+0000", 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8147000000000001e-05 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.273056+0000", 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0108999999999999e-05 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.273061+0000", 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3970000000000002e-06 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.813 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.273101+0000", 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9307999999999997e-05 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.273177+0000", 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6590999999999999e-05 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.302256+0000", 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.029079135999999998 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.814 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.305385+0000", 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0031286629999999998 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.306958+0000", 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0015727720000000001 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.338380+0000", 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.031422376000000002 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.338545+0000", 2024-09-21T10:07:25.815 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000165067 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:19.338679+0000", 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013353700000000001 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:103004 5.0 5:0d8a2d2a:::benchmark_data_smithi032_140417_object103003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:29.908305+0000", 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 55.395621577999997, 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.041517182, 2024-09-21T10:07:25.816 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 103004 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.908305+0000", 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.817 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.908305+0000", 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.908306+0000", 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.265e-06 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.908330+0000", 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3674000000000001e-05 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.818 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.908354+0000", 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4202999999999998e-05 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.908363+0000", 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1570000000000006e-06 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.913975+0000", 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0056120170000000004 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.819 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.914130+0000", 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000154637 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.914297+0000", 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016716300000000001 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.917966+0000", 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0036691319999999999 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.820 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.919186+0000", 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0012193760000000001 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.949511+0000", 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.030325652000000002 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.949703+0000", 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00019116900000000001 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.821 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:29.949822+0000", 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000119737 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:127468 5.0 5:0c541345:::benchmark_data_smithi032_140417_object127467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:53.685834+0000", 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.618093078000001, 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.052408569000000002, 2024-09-21T10:07:25.822 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 127468 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.685834+0000", 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.823 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.685834+0000", 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.685835+0000", 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.849e-06 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.685871+0000", 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5476999999999999e-05 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.685893+0000", 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1591999999999999e-05 2024-09-21T10:07:25.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.685914+0000", 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1398000000000001e-05 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.689728+0000", 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0038138920000000002 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.825 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.689871+0000", 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000143542 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.690015+0000", 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014320399999999999 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694198+0000", 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0041831610000000003 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.826 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694394+0000", 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000196116 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.738051+0000", 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.043657177999999998 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.738195+0000", 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014386800000000001 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.827 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.738242+0000", 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7292000000000001e-05 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:127476 5.0 5:07fb5b44:::benchmark_data_smithi032_140417_object127475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:53.694323+0000", 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.609603423999999, 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.044071218000000002, 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.828 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 127476 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694323+0000", 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.829 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694323+0000", 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694325+0000", 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.858e-06 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694389+0000", 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3781000000000005e-05 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.830 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694402+0000", 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.294e-05 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694409+0000", 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.452e-06 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694453+0000", 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4079000000000003e-05 2024-09-21T10:07:25.831 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694655+0000", 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00020146300000000001 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.694806+0000", 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000151037 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.699253+0000", 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0044473749999999999 2024-09-21T10:07:25.832 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.699596+0000", 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00034234199999999998 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.738292+0000", 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.038696359999999999 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.738355+0000", 2024-09-21T10:07:25.833 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3375999999999998e-05 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:53.738395+0000", 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9155e-05 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:148728 5.0 5:0b0fb098:::benchmark_data_smithi032_140417_object15337:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:07:16.506134+0000", 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.7977923269999998, 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.044897164000000003, 2024-09-21T10:07:25.834 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 148728 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.506134+0000", 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.835 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.506134+0000", 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.506136+0000", 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7379999999999999e-06 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.506142+0000", 2024-09-21T10:07:25.836 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5110000000000003e-06 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.506154+0000", 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2632e-05 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.506162+0000", 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7260000000000002e-06 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.506402+0000", 2024-09-21T10:07:25.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024011900000000001 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.506702+0000", 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029975999999999999 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.507067+0000", 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000365194 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.838 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.512318+0000", 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0052511490000000001 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.512989+0000", 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00067112300000000003 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.550674+0000", 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.037684388999999999 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.839 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.550873+0000", 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00019903400000000001 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:16.551032+0000", 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000158789 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:154216 5.0 5:0a9e9738:::benchmark_data_smithi032_140417_object20825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2024-09-21T10:07:25.840 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:07:22.499443+0000", 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 2.8044832949999998, 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.041116304999999999, 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 154216 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.499443+0000", 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.841 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.499443+0000", 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.499445+0000", 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2869999999999999e-06 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.499450+0000", 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6459999999999998e-06 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.499467+0000", 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6908000000000001e-05 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.499476+0000", 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7669999999999992e-06 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.503174+0000", 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0036979510000000001 2024-09-21T10:07:25.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.503747+0000", 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00057323699999999999 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.504215+0000", 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00046809500000000002 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.524936+0000", 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.020720358000000001 2024-09-21T10:07:25.844 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.525011+0000", 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4855000000000001e-05 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.539581+0000", 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.014570392 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.540257+0000", 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000675776 2024-09-21T10:07:25.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:22.540560+0000", 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00030303300000000002 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:154661 5.0 5:07a346b3:::benchmark_data_smithi032_140417_object21270:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:07:23.042517+0000", 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 2.261409585, 2024-09-21T10:07:25.846 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.042517078999999999, 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 154661 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.042517+0000", 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.847 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.042517+0000", 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.042519+0000", 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0990000000000001e-06 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.042525+0000", 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8649999999999998e-06 2024-09-21T10:07:25.848 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.042538+0000", 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2840000000000001e-05 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.042546+0000", 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0339999999999997e-06 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.042610+0000", 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3734999999999999e-05 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.849 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.042733+0000", 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000123251 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.042853+0000", 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000119857 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.049332+0000", 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.006478826 2024-09-21T10:07:25.850 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.050262+0000", 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00093025300000000003 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.084870+0000", 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.034608429000000003 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.084981+0000", 2024-09-21T10:07:25.851 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011042799999999999 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:07:23.085034+0000", 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3461999999999997e-05 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:07:25.852 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:07:25.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:25.853 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:25 smithi045 ceph-mon[95976]: pgmap v376: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-09-21T10:07:25.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:25.989 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:26.022 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:26.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:26.112 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:26.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:26.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:26.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:26.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:26.314 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:26.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:26.416 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:26.449 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:26.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:26.551 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.585 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:26.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:26.613 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:07:26.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:26.737 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:26.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:26.839 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:26.914 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:26.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:27.007 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:07:27.015 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:27.065 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:27.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:27.167 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:27.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:27.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:27.310 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:27.337 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:27.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:27.438 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:27.466 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:27.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:27.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:27.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:07:27.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.604 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:27.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:27.705 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:27.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:27 smithi032 ceph-mon[105748]: pgmap v377: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-21T10:07:27.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:27 smithi032 ceph-mon[101299]: pgmap v377: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-21T10:07:27.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:27.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:27.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:27 smithi045 ceph-mon[95976]: pgmap v377: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-21T10:07:27.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:27.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:27.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:27.989 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:28.027 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:28.128 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:28.163 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:28.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:28.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:28.306 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:28.407 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:28.437 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:28.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:28.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:28.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:28.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:28.676 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:28.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.716 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:28.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:28.817 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:28.848 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:28.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:28.950 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:28.988 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:28.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:29.090 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:29.132 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:07:29.132 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:07:29.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.132 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:160001 5.9 e130/112 5:9758c9c6:::benchmark_data_smithi032_140417_object26610:head v 130'9990, mlcod=130'9988)", 2024-09-21T10:07:29.132 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:29.086341+0000", 2024-09-21T10:07:29.132 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.041353084999999998, 2024-09-21T10:07:29.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.04138994, 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.086341+0000", 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.086341+0000", 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:29.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.086341+0000", 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.086345+0000", 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5140000000000003e-06 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.086352+0000", 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.046e-06 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.086358+0000", 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1229999999999999e-06 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.099457+0000", 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013098919000000001 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.099519+0000", 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2959000000000005e-05 2024-09-21T10:07:29.135 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:160005 5.9 e130/112 5:9bd64339:::benchmark_data_smithi032_140417_object26614:head v 130'9991, mlcod=130'9988)", 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:29.096177+0000", 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.031517047999999999, 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.031616641000000001, 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.096177+0000", 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.096177+0000", 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.096176+0000", 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.096186+0000", 2024-09-21T10:07:29.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0597e-05 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.096203+0000", 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6616000000000002e-05 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.096211+0000", 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9659999999999996e-06 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.114367+0000", 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018156459999999999 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.114398+0000", 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1068999999999997e-05 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:160014 5.1 5:8421e471:::benchmark_data_smithi032_140417_object26623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-21T10:07:29.139 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:29.099186+0000", 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.028508611, 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.028668025999999999, 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160014 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.099186+0000", 2024-09-21T10:07:29.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.099186+0000", 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.099184+0000", 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.099190+0000", 2024-09-21T10:07:29.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2719999999999997e-06 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.099201+0000", 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1533e-05 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.099207+0000", 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5820000000000001e-06 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:29.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.108178+0000", 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0089716350000000004 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.108298+0000", 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000119806 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.108424+0000", 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012549699999999999 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:160016 5.1 5.f13d491 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:29.100603+0000", 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.027091354000000002, 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.027318189, 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160016 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:29.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.100603+0000", 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.100603+0000", 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.100602+0000", 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.100609+0000", 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0010000000000004e-06 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.100627+0000", 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8314000000000001e-05 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.100636+0000", 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0250000000000008e-06 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:160018 5.9 e130/112 5:93dd6d30:::benchmark_data_smithi032_140417_object26627:head v 130'9992, mlcod=130'9988)", 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:29.113417+0000", 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.014277417000000001, 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014557578999999999, 2024-09-21T10:07:29.147 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.113417+0000", 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.113417+0000", 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:29.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.113415+0000", 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.113428+0000", 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2291e-05 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.113448+0000", 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.003e-05 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.149 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.113457+0000", 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0420000000000008e-06 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.123570+0000", 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010113541 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.123630+0000", 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9602999999999998e-05 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.150 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:160020 5.9 e130/112)", 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:29.116040+0000", 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.011654228000000001, 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011984013999999999, 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.116040+0000", 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.116040+0000", 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.116039+0000", 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.116047+0000", 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6580000000000002e-06 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.116061+0000", 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4174e-05 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.116069+0000", 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7139999999999993e-06 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.44543.0:160014 5.1 e130/112)", 2024-09-21T10:07:29.153 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:29.121942+0000", 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0057528170000000003, 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.006124478, 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.121942+0000", 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.154 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.121942+0000", 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.121941+0000", 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.121949+0000", 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2300000000000008e-06 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.121966+0000", 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6688e-05 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.121975+0000", 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.1589999999999999e-06 2024-09-21T10:07:29.156 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:160026 5.f e130/112 5:f7457c59:::benchmark_data_smithi032_140417_object26635:head v 130'10111, mlcod=130'10109)", 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:29.126714+0000", 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.000980105, 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013963339999999999, 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126714+0000", 2024-09-21T10:07:29.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126714+0000", 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126713+0000", 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126723+0000", 2024-09-21T10:07:29.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4269999999999997e-06 2024-09-21T10:07:29.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.159 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.159 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:29.159 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126739+0000", 2024-09-21T10:07:29.159 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5960999999999998e-05 2024-09-21T10:07:29.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.161 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.161 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126747+0000", 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1810000000000001e-06 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.128064+0000", 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013172909999999999 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.128084+0000", 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9765000000000001e-05 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.162 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:160030 5.e 5:7457bb9e:::benchmark_data_smithi032_140417_object26639:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:29.126836+0000", 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00085868900000000005, 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013333609999999999, 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160030 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.163 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126836+0000", 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126836+0000", 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126834+0000", 2024-09-21T10:07:29.164 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126843+0000", 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4299999999999995e-06 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126855+0000", 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1513e-05 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.165 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126864+0000", 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6640000000000004e-06 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.126921+0000", 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7025999999999999e-05 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.127063+0000", 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000141932 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:29.166 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:29.127199+0000", 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013609900000000001 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 9 2024-09-21T10:07:29.167 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:07:29.200 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:29.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:29.302 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:29.366 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:29.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:29.471 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:29.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.515 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:29.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:29.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.651 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:29.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:29.729 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:29 smithi032 ceph-mon[105748]: pgmap v378: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-09-21T10:07:29.729 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:29 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:29.729 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:07:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:07:29] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.51.0" 2024-09-21T10:07:29.730 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:29 smithi032 ceph-mon[101299]: pgmap v378: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-09-21T10:07:29.730 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:29 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:29.747 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:29 smithi045 ceph-mon[95976]: pgmap v378: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-09-21T10:07:29.747 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:29 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:29.752 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:29.792 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:29.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:29.894 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:29.942 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:29.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:30.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:30.092 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:30.102 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:07:30.194 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:30.239 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:30.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:30.269 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:30.275 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:07:30.275 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:30.321 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:07:30.322 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:07:30.337 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:07:30.338 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:30.341 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:30.403 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:07:30.403 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:30.410 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:30.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:30.438 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:07:30.438 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:30.478 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:07:30.478 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:07:30.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:30.517 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:07:30.517 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:30.554 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:07:30.554 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:30.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.565 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:30.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:30.588 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:30 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/194395901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:30.588 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:30 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/194395901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:30.600 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:07:30.600 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:30.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:30.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.731 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:30.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:30 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/194395901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:30.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:30.874 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:30.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:30.976 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:31.009 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:31.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:31.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.147 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:31.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:31.182 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:07:31.248 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:31.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.317 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:31.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:31.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:31.440 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":379,"stamp":"2024-09-21T10:07:30.184143+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7047026685,"num_objects":107485,"num_object_clones":0,"num_object_copies":322455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107485,"num_whiteouts":0,"num_read":6391,"num_read_kb":13846,"num_write":160444,"num_write_kb":8551874,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147203,"ondisk_log_size":147203,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21398328,"kb_used_data":20706360,"kb_used_omap":35,"kb_used_meta":691036,"kb_avail":728562888,"statfs":{"total":767960285184,"available":746048397312,"internally_reserved":0,"allocated":21203312640,"data_stored":21197887519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":707621582},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[11,3,6,4,11,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-587464704,"num_objects":-8964,"num_object_clones":0,"num_object_copies":-26892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8964,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008090"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":116,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.481145+0000","last_change":"2024-09-21T10:03:53.843548+0000","last_active":"2024-09-21T10:07:23.481145+0000","last_peered":"2024-09-21T10:07:23.481145+0000","last_clean":"2024-09-21T10:07:23.481145+0000","last_became_active":"2024-09-21T10:03:53.843182+0000","last_became_peered":"2024-09-21T10:03:53.843182+0000","last_unstale":"2024-09-21T10:07:23.481145+0000","last_undegraded":"2024-09-21T10:07:23.481145+0000","last_fullsized":"2024-09-21T10:07:23.481145+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:09.464209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":96,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.509596+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:07:09.509596+0000","last_peered":"2024-09-21T10:07:09.509596+0000","last_clean":"2024-09-21T10:07:09.509596+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:07:09.509596+0000","last_undegraded":"2024-09-21T10:07:09.509596+0000","last_fullsized":"2024-09-21T10:07:09.509596+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:14.278394+0000","last_change":"2024-09-21T10:03:53.839987+0000","last_active":"2024-09-21T10:07:14.278394+0000","last_peered":"2024-09-21T10:07:14.278394+0000","last_clean":"2024-09-21T10:07:14.278394+0000","last_became_active":"2024-09-21T10:03:53.839664+0000","last_became_peered":"2024-09-21T10:03:53.839664+0000","last_unstale":"2024-09-21T10:07:14.278394+0000","last_undegraded":"2024-09-21T10:07:14.278394+0000","last_fullsized":"2024-09-21T10:07:14.278394+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:51.726392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.517815+0000","last_change":"2024-09-21T10:03:53.837496+0000","last_active":"2024-09-21T10:07:23.517815+0000","last_peered":"2024-09-21T10:07:23.517815+0000","last_clean":"2024-09-21T10:07:23.517815+0000","last_became_active":"2024-09-21T10:03:53.837166+0000","last_became_peered":"2024-09-21T10:03:53.837166+0000","last_unstale":"2024-09-21T10:07:23.517815+0000","last_undegraded":"2024-09-21T10:07:23.517815+0000","last_fullsized":"2024-09-21T10:07:23.517815+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:45.795571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":108,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.522057+0000","last_change":"2024-09-21T10:03:53.844990+0000","last_active":"2024-09-21T10:07:23.522057+0000","last_peered":"2024-09-21T10:07:23.522057+0000","last_clean":"2024-09-21T10:07:23.522057+0000","last_became_active":"2024-09-21T10:03:53.844664+0000","last_became_peered":"2024-09-21T10:03:53.844664+0000","last_unstale":"2024-09-21T10:07:23.522057+0000","last_undegraded":"2024-09-21T10:07:23.522057+0000","last_fullsized":"2024-09-21T10:07:23.522057+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:01.031522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":79,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.514381+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:07:09.514381+0000","last_peered":"2024-09-21T10:07:09.514381+0000","last_clean":"2024-09-21T10:07:09.514381+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:07:09.514381+0000","last_undegraded":"2024-09-21T10:07:09.514381+0000","last_fullsized":"2024-09-21T10:07:09.514381+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":116,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.479938+0000","last_change":"2024-09-21T10:03:53.845785+0000","last_active":"2024-09-21T10:07:23.479938+0000","last_peered":"2024-09-21T10:07:23.479938+0000","last_clean":"2024-09-21T10:07:23.479938+0000","last_became_active":"2024-09-21T10:03:53.845303+0000","last_became_peered":"2024-09-21T10:03:53.845303+0000","last_unstale":"2024-09-21T10:07:23.479938+0000","last_undegraded":"2024-09-21T10:07:23.479938+0000","last_fullsized":"2024-09-21T10:07:23.479938+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:04.976007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":102,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.515940+0000","last_change":"2024-09-21T10:02:12.838786+0000","last_active":"2024-09-21T10:07:09.515940+0000","last_peered":"2024-09-21T10:07:09.515940+0000","last_clean":"2024-09-21T10:07:09.515940+0000","last_became_active":"2024-09-21T10:02:12.838242+0000","last_became_peered":"2024-09-21T10:02:12.838242+0000","last_unstale":"2024-09-21T10:07:09.515940+0000","last_undegraded":"2024-09-21T10:07:09.515940+0000","last_fullsized":"2024-09-21T10:07:09.515940+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:31.071136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"130'9993","reported_seq":10019,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:29.143013+0000","last_change":"2024-09-21T10:05:19.211627+0000","last_active":"2024-09-21T10:07:29.143013+0000","last_peered":"2024-09-21T10:07:29.143013+0000","last_clean":"2024-09-21T10:07:29.143013+0000","last_became_active":"2024-09-21T10:05:19.211261+0000","last_became_peered":"2024-09-21T10:05:19.211261+0000","last_unstale":"2024-09-21T10:07:29.143013+0000","last_undegraded":"2024-09-21T10:07:29.143013+0000","last_fullsized":"2024-09-21T10:07:29.143013+0000","mapping_epoch":112,"log_start":"117'1400","ondisk_log_start":"117'1400","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8593,"log_dups_size":0,"ondisk_log_size":8593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:10.360788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439025664,"num_objects":6699,"num_object_clones":0,"num_object_copies":20097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9993,"num_write_kb":534144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":95,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.509687+0000","last_change":"2024-09-21T10:02:12.840334+0000","last_active":"2024-09-21T10:07:09.509687+0000","last_peered":"2024-09-21T10:07:09.509687+0000","last_clean":"2024-09-21T10:07:09.509687+0000","last_became_active":"2024-09-21T10:02:12.839928+0000","last_became_peered":"2024-09-21T10:02:12.839928+0000","last_unstale":"2024-09-21T10:07:09.509687+0000","last_undegraded":"2024-09-21T10:07:09.509687+0000","last_fullsized":"2024-09-21T10:07:09.509687+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:38.593644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":78,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.507696+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T10:07:09.507696+0000","last_peered":"2024-09-21T10:07:09.507696+0000","last_clean":"2024-09-21T10:07:09.507696+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T10:07:09.507696+0000","last_undegraded":"2024-09-21T10:07:09.507696+0000","last_fullsized":"2024-09-21T10:07:09.507696+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"130'9933","reported_seq":9980,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:28.045991+0000","last_change":"2024-09-21T10:05:19.213525+0000","last_active":"2024-09-21T10:07:28.045991+0000","last_peered":"2024-09-21T10:07:28.045991+0000","last_clean":"2024-09-21T10:07:28.045991+0000","last_became_active":"2024-09-21T10:05:19.212434+0000","last_became_peered":"2024-09-21T10:05:19.212434+0000","last_unstale":"2024-09-21T10:07:28.045991+0000","last_undegraded":"2024-09-21T10:07:28.045991+0000","last_fullsized":"2024-09-21T10:07:28.045991+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9933,"log_dups_size":0,"ondisk_log_size":9933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:35.483451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436142080,"num_objects":6655,"num_object_clones":0,"num_object_copies":19965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9933,"num_write_kb":530816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":96,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.516652+0000","last_change":"2024-09-21T10:02:46.662169+0000","last_active":"2024-09-21T10:07:09.516652+0000","last_peered":"2024-09-21T10:07:09.516652+0000","last_clean":"2024-09-21T10:07:09.516652+0000","last_became_active":"2024-09-21T10:02:46.661721+0000","last_became_peered":"2024-09-21T10:02:46.661721+0000","last_unstale":"2024-09-21T10:07:09.516652+0000","last_undegraded":"2024-09-21T10:07:09.516652+0000","last_fullsized":"2024-09-21T10:07:09.516652+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:10.414152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":108,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504989+0000","last_change":"2024-09-21T10:03:53.840738+0000","last_active":"2024-09-21T10:07:09.504989+0000","last_peered":"2024-09-21T10:07:09.504989+0000","last_clean":"2024-09-21T10:07:09.504989+0000","last_became_active":"2024-09-21T10:03:53.840363+0000","last_became_peered":"2024-09-21T10:03:53.840363+0000","last_unstale":"2024-09-21T10:07:09.504989+0000","last_undegraded":"2024-09-21T10:07:09.504989+0000","last_fullsized":"2024-09-21T10:07:09.504989+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:26.371181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"130'9902","reported_seq":9949,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:28.039738+0000","last_change":"2024-09-21T10:05:19.213181+0000","last_active":"2024-09-21T10:07:28.039738+0000","last_peered":"2024-09-21T10:07:28.039738+0000","last_clean":"2024-09-21T10:07:28.039738+0000","last_became_active":"2024-09-21T10:05:19.212895+0000","last_became_peered":"2024-09-21T10:05:19.212895+0000","last_unstale":"2024-09-21T10:07:28.039738+0000","last_undegraded":"2024-09-21T10:07:28.039738+0000","last_fullsized":"2024-09-21T10:07:28.039738+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9902,"log_dups_size":0,"ondisk_log_size":9902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:40:31.162520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444203008,"num_objects":6778,"num_object_clones":0,"num_object_copies":20334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9902,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":80,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.514631+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T10:07:09.514631+0000","last_peered":"2024-09-21T10:07:09.514631+0000","last_clean":"2024-09-21T10:07:09.514631+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T10:07:09.514631+0000","last_undegraded":"2024-09-21T10:07:09.514631+0000","last_fullsized":"2024-09-21T10:07:09.514631+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.516238+0000","last_change":"2024-09-21T10:02:46.663127+0000","last_active":"2024-09-21T10:07:09.516238+0000","last_peered":"2024-09-21T10:07:09.516238+0000","last_clean":"2024-09-21T10:07:09.516238+0000","last_became_active":"2024-09-21T10:02:46.662791+0000","last_became_peered":"2024-09-21T10:02:46.662791+0000","last_unstale":"2024-09-21T10:07:09.516238+0000","last_undegraded":"2024-09-21T10:07:09.516238+0000","last_fullsized":"2024-09-21T10:07:09.516238+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:59.990122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"130'9872","reported_seq":9919,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:27.989727+0000","last_change":"2024-09-21T10:05:19.212279+0000","last_active":"2024-09-21T10:07:27.989727+0000","last_peered":"2024-09-21T10:07:27.989727+0000","last_clean":"2024-09-21T10:07:27.989727+0000","last_became_active":"2024-09-21T10:05:19.211960+0000","last_became_peered":"2024-09-21T10:05:19.211960+0000","last_unstale":"2024-09-21T10:07:27.989727+0000","last_undegraded":"2024-09-21T10:07:27.989727+0000","last_fullsized":"2024-09-21T10:07:27.989727+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9872,"log_dups_size":0,"ondisk_log_size":9872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:40.251441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":447086592,"num_objects":6822,"num_object_clones":0,"num_object_copies":20466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9872,"num_write_kb":534208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.508241+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:07:09.508241+0000","last_peered":"2024-09-21T10:07:09.508241+0000","last_clean":"2024-09-21T10:07:09.508241+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:07:09.508241+0000","last_undegraded":"2024-09-21T10:07:09.508241+0000","last_fullsized":"2024-09-21T10:07:09.508241+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":108,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.517577+0000","last_change":"2024-09-21T10:02:46.662651+0000","last_active":"2024-09-21T10:07:23.517577+0000","last_peered":"2024-09-21T10:07:23.517577+0000","last_clean":"2024-09-21T10:07:23.517577+0000","last_became_active":"2024-09-21T10:02:46.662304+0000","last_became_peered":"2024-09-21T10:02:46.662304+0000","last_unstale":"2024-09-21T10:07:23.517577+0000","last_undegraded":"2024-09-21T10:07:23.517577+0000","last_fullsized":"2024-09-21T10:07:23.517577+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:06.746929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"130'10112","reported_seq":10159,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:28.052185+0000","last_change":"2024-09-21T10:05:19.211133+0000","last_active":"2024-09-21T10:07:28.052185+0000","last_peered":"2024-09-21T10:07:28.052185+0000","last_clean":"2024-09-21T10:07:28.052185+0000","last_became_active":"2024-09-21T10:05:19.210822+0000","last_became_peered":"2024-09-21T10:05:19.210822+0000","last_unstale":"2024-09-21T10:07:28.052185+0000","last_undegraded":"2024-09-21T10:07:28.052185+0000","last_fullsized":"2024-09-21T10:07:28.052185+0000","mapping_epoch":112,"log_start":"116'100","ondisk_log_start":"116'100","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10012,"log_dups_size":0,"ondisk_log_size":10012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:02.399086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444923928,"num_objects":6790,"num_object_clones":0,"num_object_copies":20370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6790,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10112,"num_write_kb":540801,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":114,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.479820+0000","last_change":"2024-09-21T10:03:53.840511+0000","last_active":"2024-09-21T10:07:23.479820+0000","last_peered":"2024-09-21T10:07:23.479820+0000","last_clean":"2024-09-21T10:07:23.479820+0000","last_became_active":"2024-09-21T10:03:53.840136+0000","last_became_peered":"2024-09-21T10:03:53.840136+0000","last_unstale":"2024-09-21T10:07:23.479820+0000","last_undegraded":"2024-09-21T10:07:23.479820+0000","last_fullsized":"2024-09-21T10:07:23.479820+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:24.765990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504261+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:07:09.504261+0000","last_peered":"2024-09-21T10:07:09.504261+0000","last_clean":"2024-09-21T10:07:09.504261+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:07:09.504261+0000","last_undegraded":"2024-09-21T10:07:09.504261+0000","last_fullsized":"2024-09-21T10:07:09.504261+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"130'9956","reported_seq":10003,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:29.502821+0000","last_change":"2024-09-21T10:05:19.214310+0000","last_active":"2024-09-21T10:07:29.502821+0000","last_peered":"2024-09-21T10:07:29.502821+0000","last_clean":"2024-09-21T10:07:29.502821+0000","last_became_active":"2024-09-21T10:05:19.213979+0000","last_became_peered":"2024-09-21T10:05:19.213979+0000","last_unstale":"2024-09-21T10:07:29.502821+0000","last_undegraded":"2024-09-21T10:07:29.502821+0000","last_fullsized":"2024-09-21T10:07:29.502821+0000","mapping_epoch":112,"log_start":"117'1800","ondisk_log_start":"117'1800","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:05.291457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435552256,"num_objects":6646,"num_object_clones":0,"num_object_copies":19938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9956,"num_write_kb":531264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":94,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.516322+0000","last_change":"2024-09-21T10:02:46.663900+0000","last_active":"2024-09-21T10:07:09.516322+0000","last_peered":"2024-09-21T10:07:09.516322+0000","last_clean":"2024-09-21T10:07:09.516322+0000","last_became_active":"2024-09-21T10:02:46.663594+0000","last_became_peered":"2024-09-21T10:02:46.663594+0000","last_unstale":"2024-09-21T10:07:09.516322+0000","last_undegraded":"2024-09-21T10:07:09.516322+0000","last_fullsized":"2024-09-21T10:07:09.516322+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:24.140061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.506308+0000","last_change":"2024-09-21T10:03:19.765623+0000","last_active":"2024-09-21T10:07:09.506308+0000","last_peered":"2024-09-21T10:07:09.506308+0000","last_clean":"2024-09-21T10:07:09.506308+0000","last_became_active":"2024-09-21T10:03:19.765227+0000","last_became_peered":"2024-09-21T10:03:19.765227+0000","last_unstale":"2024-09-21T10:07:09.506308+0000","last_undegraded":"2024-09-21T10:07:09.506308+0000","last_fullsized":"2024-09-21T10:07:09.506308+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:52.227826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"130'10118","reported_seq":10145,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:29.328633+0000","last_change":"2024-09-21T10:05:19.212082+0000","last_active":"2024-09-21T10:07:29.328633+0000","last_peered":"2024-09-21T10:07:29.328633+0000","last_clean":"2024-09-21T10:07:29.328633+0000","last_became_active":"2024-09-21T10:05:19.211673+0000","last_became_peered":"2024-09-21T10:05:19.211673+0000","last_unstale":"2024-09-21T10:07:29.328633+0000","last_undegraded":"2024-09-21T10:07:29.328633+0000","last_fullsized":"2024-09-21T10:07:29.328633+0000","mapping_epoch":112,"log_start":"116'100","ondisk_log_start":"116'100","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10018,"log_dups_size":0,"ondisk_log_size":10018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:12.775587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437649408,"num_objects":6678,"num_object_clones":0,"num_object_copies":20034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10118,"num_write_kb":537472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":94,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.505188+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:07:09.505188+0000","last_peered":"2024-09-21T10:07:09.505188+0000","last_clean":"2024-09-21T10:07:09.505188+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:07:09.505188+0000","last_undegraded":"2024-09-21T10:07:09.505188+0000","last_fullsized":"2024-09-21T10:07:09.505188+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.509614+0000","last_change":"2024-09-21T10:02:46.664014+0000","last_active":"2024-09-21T10:07:09.509614+0000","last_peered":"2024-09-21T10:07:09.509614+0000","last_clean":"2024-09-21T10:07:09.509614+0000","last_became_active":"2024-09-21T10:02:46.663640+0000","last_became_peered":"2024-09-21T10:02:46.663640+0000","last_unstale":"2024-09-21T10:07:09.509614+0000","last_undegraded":"2024-09-21T10:07:09.509614+0000","last_fullsized":"2024-09-21T10:07:09.509614+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:03.733462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"130'10041","reported_seq":10068,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:29.159209+0000","last_change":"2024-09-21T10:05:19.214684+0000","last_active":"2024-09-21T10:07:29.159209+0000","last_peered":"2024-09-21T10:07:29.159209+0000","last_clean":"2024-09-21T10:07:29.159209+0000","last_became_active":"2024-09-21T10:05:19.214356+0000","last_became_peered":"2024-09-21T10:05:19.214356+0000","last_unstale":"2024-09-21T10:07:29.159209+0000","last_undegraded":"2024-09-21T10:07:29.159209+0000","last_fullsized":"2024-09-21T10:07:29.159209+0000","mapping_epoch":112,"log_start":"118'2700","ondisk_log_start":"118'2700","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":7341,"log_dups_size":0,"ondisk_log_size":7341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:04.876232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435486720,"num_objects":6645,"num_object_clones":0,"num_object_copies":19935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10041,"num_write_kb":533952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":117,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.480092+0000","last_change":"2024-09-21T10:03:53.845620+0000","last_active":"2024-09-21T10:07:23.480092+0000","last_peered":"2024-09-21T10:07:23.480092+0000","last_clean":"2024-09-21T10:07:23.480092+0000","last_became_active":"2024-09-21T10:03:53.845290+0000","last_became_peered":"2024-09-21T10:03:53.845290+0000","last_unstale":"2024-09-21T10:07:23.480092+0000","last_undegraded":"2024-09-21T10:07:23.480092+0000","last_fullsized":"2024-09-21T10:07:23.480092+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:20.076623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.502781+0000","last_change":"2024-09-21T10:03:53.846606+0000","last_active":"2024-09-21T10:07:23.502781+0000","last_peered":"2024-09-21T10:07:23.502781+0000","last_clean":"2024-09-21T10:07:23.502781+0000","last_became_active":"2024-09-21T10:03:53.846245+0000","last_became_peered":"2024-09-21T10:03:53.846245+0000","last_unstale":"2024-09-21T10:07:23.502781+0000","last_undegraded":"2024-09-21T10:07:23.502781+0000","last_fullsized":"2024-09-21T10:07:23.502781+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:20.745470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"130'10083","reported_seq":10111,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:29.092145+0000","last_change":"2024-09-21T10:05:19.210796+0000","last_active":"2024-09-21T10:07:29.092145+0000","last_peered":"2024-09-21T10:07:29.092145+0000","last_clean":"2024-09-21T10:07:29.092145+0000","last_became_active":"2024-09-21T10:05:19.210466+0000","last_became_peered":"2024-09-21T10:05:19.210466+0000","last_unstale":"2024-09-21T10:07:29.092145+0000","last_undegraded":"2024-09-21T10:07:29.092145+0000","last_fullsized":"2024-09-21T10:07:29.092145+0000","mapping_epoch":112,"log_start":"117'1500","ondisk_log_start":"117'1500","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8583,"log_dups_size":0,"ondisk_log_size":8583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:11.178437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437977088,"num_objects":6683,"num_object_clones":0,"num_object_copies":20049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10083,"num_write_kb":536512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":426,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.509502+0000","last_change":"2024-09-21T10:02:12.840225+0000","last_active":"2024-09-21T10:07:09.509502+0000","last_peered":"2024-09-21T10:07:09.509502+0000","last_clean":"2024-09-21T10:07:09.509502+0000","last_became_active":"2024-09-21T10:02:12.839813+0000","last_became_peered":"2024-09-21T10:02:12.839813+0000","last_unstale":"2024-09-21T10:07:09.509502+0000","last_undegraded":"2024-09-21T10:07:09.509502+0000","last_fullsized":"2024-09-21T10:07:09.509502+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:36.630881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":444,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.508080+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:07:09.508080+0000","last_peered":"2024-09-21T10:07:09.508080+0000","last_clean":"2024-09-21T10:07:09.508080+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:07:09.508080+0000","last_undegraded":"2024-09-21T10:07:09.508080+0000","last_fullsized":"2024-09-21T10:07:09.508080+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.482017+0000","last_change":"2024-09-21T10:03:53.843344+0000","last_active":"2024-09-21T10:07:23.482017+0000","last_peered":"2024-09-21T10:07:23.482017+0000","last_clean":"2024-09-21T10:07:23.482017+0000","last_became_active":"2024-09-21T10:03:53.842304+0000","last_became_peered":"2024-09-21T10:03:53.842304+0000","last_unstale":"2024-09-21T10:07:23.482017+0000","last_undegraded":"2024-09-21T10:07:23.482017+0000","last_fullsized":"2024-09-21T10:07:23.482017+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:31.221735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":79,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.516441+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:07:09.516441+0000","last_peered":"2024-09-21T10:07:09.516441+0000","last_clean":"2024-09-21T10:07:09.516441+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:07:09.516441+0000","last_undegraded":"2024-09-21T10:07:09.516441+0000","last_fullsized":"2024-09-21T10:07:09.516441+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"130'10102","reported_seq":10128,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:29.354204+0000","last_change":"2024-09-21T10:05:19.211638+0000","last_active":"2024-09-21T10:07:29.354204+0000","last_peered":"2024-09-21T10:07:29.354204+0000","last_clean":"2024-09-21T10:07:29.354204+0000","last_became_active":"2024-09-21T10:05:19.211301+0000","last_became_peered":"2024-09-21T10:05:19.211301+0000","last_unstale":"2024-09-21T10:07:29.354204+0000","last_undegraded":"2024-09-21T10:07:29.354204+0000","last_fullsized":"2024-09-21T10:07:29.354204+0000","mapping_epoch":112,"log_start":"116'100","ondisk_log_start":"116'100","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:53.839733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442368000,"num_objects":6750,"num_object_clones":0,"num_object_copies":20250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10102,"num_write_kb":539264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":590,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.502825+0000","last_change":"2024-09-21T10:02:46.663394+0000","last_active":"2024-09-21T10:07:23.502825+0000","last_peered":"2024-09-21T10:07:23.502825+0000","last_clean":"2024-09-21T10:07:23.502825+0000","last_became_active":"2024-09-21T10:02:46.662879+0000","last_became_peered":"2024-09-21T10:02:46.662879+0000","last_unstale":"2024-09-21T10:07:23.502825+0000","last_undegraded":"2024-09-21T10:07:23.502825+0000","last_fullsized":"2024-09-21T10:07:23.502825+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.125795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.516897+0000","last_change":"2024-09-21T10:03:53.835329+0000","last_active":"2024-09-21T10:07:23.516897+0000","last_peered":"2024-09-21T10:07:23.516897+0000","last_clean":"2024-09-21T10:07:23.516897+0000","last_became_active":"2024-09-21T10:03:53.834934+0000","last_became_peered":"2024-09-21T10:03:53.834934+0000","last_unstale":"2024-09-21T10:07:23.516897+0000","last_undegraded":"2024-09-21T10:07:23.516897+0000","last_fullsized":"2024-09-21T10:07:23.516897+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:38.355817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":119,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.522093+0000","last_change":"2024-09-21T10:03:19.770016+0000","last_active":"2024-09-21T10:07:23.522093+0000","last_peered":"2024-09-21T10:07:23.522093+0000","last_clean":"2024-09-21T10:07:23.522093+0000","last_became_active":"2024-09-21T10:03:19.768866+0000","last_became_peered":"2024-09-21T10:03:19.768866+0000","last_unstale":"2024-09-21T10:07:23.522093+0000","last_undegraded":"2024-09-21T10:07:23.522093+0000","last_fullsized":"2024-09-21T10:07:23.522093+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:25.068940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"130'9859","reported_seq":9906,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:28.040069+0000","last_change":"2024-09-21T10:05:19.210830+0000","last_active":"2024-09-21T10:07:28.040069+0000","last_peered":"2024-09-21T10:07:28.040069+0000","last_clean":"2024-09-21T10:07:28.040069+0000","last_became_active":"2024-09-21T10:05:19.210525+0000","last_became_peered":"2024-09-21T10:05:19.210525+0000","last_unstale":"2024-09-21T10:07:28.040069+0000","last_undegraded":"2024-09-21T10:07:28.040069+0000","last_fullsized":"2024-09-21T10:07:28.040069+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9859,"log_dups_size":0,"ondisk_log_size":9859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:44.771575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438370304,"num_objects":6689,"num_object_clones":0,"num_object_copies":20067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9859,"num_write_kb":529536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":615,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.494131+0000","last_change":"2024-09-21T10:03:53.840322+0000","last_active":"2024-09-21T10:07:23.494131+0000","last_peered":"2024-09-21T10:07:23.494131+0000","last_clean":"2024-09-21T10:07:23.494131+0000","last_became_active":"2024-09-21T10:03:53.839753+0000","last_became_peered":"2024-09-21T10:03:53.839753+0000","last_unstale":"2024-09-21T10:07:23.494131+0000","last_undegraded":"2024-09-21T10:07:23.494131+0000","last_fullsized":"2024-09-21T10:07:23.494131+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:26.364888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.499999+0000","last_change":"2024-09-21T10:03:19.765402+0000","last_active":"2024-09-21T10:07:23.499999+0000","last_peered":"2024-09-21T10:07:23.499999+0000","last_clean":"2024-09-21T10:07:23.499999+0000","last_became_active":"2024-09-21T10:03:19.765036+0000","last_became_peered":"2024-09-21T10:03:19.765036+0000","last_unstale":"2024-09-21T10:07:23.499999+0000","last_undegraded":"2024-09-21T10:07:23.499999+0000","last_fullsized":"2024-09-21T10:07:23.499999+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:32.230377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":131,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.502640+0000","last_change":"2024-09-21T10:02:46.660814+0000","last_active":"2024-09-21T10:07:23.502640+0000","last_peered":"2024-09-21T10:07:23.502640+0000","last_clean":"2024-09-21T10:07:23.502640+0000","last_became_active":"2024-09-21T10:02:46.660325+0000","last_became_peered":"2024-09-21T10:02:46.660325+0000","last_unstale":"2024-09-21T10:07:23.502640+0000","last_undegraded":"2024-09-21T10:07:23.502640+0000","last_fullsized":"2024-09-21T10:07:23.502640+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:44.427243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"130'9808","reported_seq":9856,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:26.100502+0000","last_change":"2024-09-21T10:05:19.213938+0000","last_active":"2024-09-21T10:07:26.100502+0000","last_peered":"2024-09-21T10:07:26.100502+0000","last_clean":"2024-09-21T10:07:26.100502+0000","last_became_active":"2024-09-21T10:05:19.213559+0000","last_became_peered":"2024-09-21T10:05:19.213559+0000","last_unstale":"2024-09-21T10:07:26.100502+0000","last_undegraded":"2024-09-21T10:07:26.100502+0000","last_fullsized":"2024-09-21T10:07:26.100502+0000","mapping_epoch":112,"log_start":"117'900","ondisk_log_start":"117'900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8908,"log_dups_size":0,"ondisk_log_size":8908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:25.094755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448397312,"num_objects":6842,"num_object_clones":0,"num_object_copies":20526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9808,"num_write_kb":532800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":832,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.514343+0000","last_change":"2024-09-21T10:02:46.657107+0000","last_active":"2024-09-21T10:07:09.514343+0000","last_peered":"2024-09-21T10:07:09.514343+0000","last_clean":"2024-09-21T10:07:09.514343+0000","last_became_active":"2024-09-21T10:02:46.656633+0000","last_became_peered":"2024-09-21T10:02:46.656633+0000","last_unstale":"2024-09-21T10:07:09.514343+0000","last_undegraded":"2024-09-21T10:07:09.514343+0000","last_fullsized":"2024-09-21T10:07:09.514343+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:42:18.534832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":95,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504630+0000","last_change":"2024-09-21T10:02:12.840581+0000","last_active":"2024-09-21T10:07:09.504630+0000","last_peered":"2024-09-21T10:07:09.504630+0000","last_clean":"2024-09-21T10:07:09.504630+0000","last_became_active":"2024-09-21T10:02:12.840033+0000","last_became_peered":"2024-09-21T10:02:12.840033+0000","last_unstale":"2024-09-21T10:07:09.504630+0000","last_undegraded":"2024-09-21T10:07:09.504630+0000","last_fullsized":"2024-09-21T10:07:09.504630+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:22.449586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.509812+0000","last_change":"2024-09-21T10:03:53.843551+0000","last_active":"2024-09-21T10:07:23.509812+0000","last_peered":"2024-09-21T10:07:23.509812+0000","last_clean":"2024-09-21T10:07:23.509812+0000","last_became_active":"2024-09-21T10:03:53.843007+0000","last_became_peered":"2024-09-21T10:03:53.843007+0000","last_unstale":"2024-09-21T10:07:23.509812+0000","last_undegraded":"2024-09-21T10:07:23.509812+0000","last_fullsized":"2024-09-21T10:07:23.509812+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:11.757447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"130'9854","reported_seq":9898,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:26.108260+0000","last_change":"2024-09-21T10:05:19.212703+0000","last_active":"2024-09-21T10:07:26.108260+0000","last_peered":"2024-09-21T10:07:26.108260+0000","last_clean":"2024-09-21T10:07:26.108260+0000","last_became_active":"2024-09-21T10:05:19.212420+0000","last_became_peered":"2024-09-21T10:05:19.212420+0000","last_unstale":"2024-09-21T10:07:26.108260+0000","last_undegraded":"2024-09-21T10:07:26.108260+0000","last_fullsized":"2024-09-21T10:07:26.108260+0000","mapping_epoch":112,"log_start":"117'1000","ondisk_log_start":"117'1000","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8854,"log_dups_size":0,"ondisk_log_size":8854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:43.062243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446300160,"num_objects":6810,"num_object_clones":0,"num_object_copies":20430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9854,"num_write_kb":533248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":110,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.508408+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:07:09.508408+0000","last_peered":"2024-09-21T10:07:09.508408+0000","last_clean":"2024-09-21T10:07:09.508408+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:07:09.508408+0000","last_undegraded":"2024-09-21T10:07:09.508408+0000","last_fullsized":"2024-09-21T10:07:09.508408+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":571,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.509924+0000","last_change":"2024-09-21T10:02:46.662410+0000","last_active":"2024-09-21T10:07:23.509924+0000","last_peered":"2024-09-21T10:07:23.509924+0000","last_clean":"2024-09-21T10:07:23.509924+0000","last_became_active":"2024-09-21T10:02:46.661506+0000","last_became_peered":"2024-09-21T10:02:46.661506+0000","last_unstale":"2024-09-21T10:07:23.509924+0000","last_undegraded":"2024-09-21T10:07:23.509924+0000","last_fullsized":"2024-09-21T10:07:23.509924+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:40:33.566844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":87,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.499638+0000","last_change":"2024-09-21T10:02:46.665713+0000","last_active":"2024-09-21T10:07:23.499638+0000","last_peered":"2024-09-21T10:07:23.499638+0000","last_clean":"2024-09-21T10:07:23.499638+0000","last_became_active":"2024-09-21T10:02:46.665316+0000","last_became_peered":"2024-09-21T10:02:46.665316+0000","last_unstale":"2024-09-21T10:07:23.499638+0000","last_undegraded":"2024-09-21T10:07:23.499638+0000","last_fullsized":"2024-09-21T10:07:23.499638+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.951202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"130'9932","reported_seq":9960,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:29.188843+0000","last_change":"2024-09-21T10:05:19.214304+0000","last_active":"2024-09-21T10:07:29.188843+0000","last_peered":"2024-09-21T10:07:29.188843+0000","last_clean":"2024-09-21T10:07:29.188843+0000","last_became_active":"2024-09-21T10:05:19.213946+0000","last_became_peered":"2024-09-21T10:05:19.213946+0000","last_unstale":"2024-09-21T10:07:29.188843+0000","last_undegraded":"2024-09-21T10:07:29.188843+0000","last_fullsized":"2024-09-21T10:07:29.188843+0000","mapping_epoch":112,"log_start":"116'200","ondisk_log_start":"116'200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9732,"log_dups_size":0,"ondisk_log_size":9732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:18.695450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428212224,"num_objects":6534,"num_object_clones":0,"num_object_copies":19602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9932,"num_write_kb":526912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.510079+0000","last_change":"2024-09-21T10:03:19.768187+0000","last_active":"2024-09-21T10:07:23.510079+0000","last_peered":"2024-09-21T10:07:23.510079+0000","last_clean":"2024-09-21T10:07:23.510079+0000","last_became_active":"2024-09-21T10:03:19.767801+0000","last_became_peered":"2024-09-21T10:03:19.767801+0000","last_unstale":"2024-09-21T10:07:23.510079+0000","last_undegraded":"2024-09-21T10:07:23.510079+0000","last_fullsized":"2024-09-21T10:07:23.510079+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:38.042481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":532,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.514703+0000","last_change":"2024-09-21T10:02:12.840296+0000","last_active":"2024-09-21T10:07:09.514703+0000","last_peered":"2024-09-21T10:07:09.514703+0000","last_clean":"2024-09-21T10:07:09.514703+0000","last_became_active":"2024-09-21T10:02:12.839881+0000","last_became_peered":"2024-09-21T10:02:12.839881+0000","last_unstale":"2024-09-21T10:07:09.514703+0000","last_undegraded":"2024-09-21T10:07:09.514703+0000","last_fullsized":"2024-09-21T10:07:09.514703+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:49.166865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.517490+0000","last_change":"2024-09-21T10:02:12.845158+0000","last_active":"2024-09-21T10:07:23.517490+0000","last_peered":"2024-09-21T10:07:23.517490+0000","last_clean":"2024-09-21T10:07:23.517490+0000","last_became_active":"2024-09-21T10:02:12.843399+0000","last_became_peered":"2024-09-21T10:02:12.843399+0000","last_unstale":"2024-09-21T10:07:23.517490+0000","last_undegraded":"2024-09-21T10:07:23.517490+0000","last_fullsized":"2024-09-21T10:07:23.517490+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:27.552491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"130'9840","reported_seq":9867,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:29.196739+0000","last_change":"2024-09-21T10:05:19.214110+0000","last_active":"2024-09-21T10:07:29.196739+0000","last_peered":"2024-09-21T10:07:29.196739+0000","last_clean":"2024-09-21T10:07:29.196739+0000","last_became_active":"2024-09-21T10:05:19.213716+0000","last_became_peered":"2024-09-21T10:05:19.213716+0000","last_unstale":"2024-09-21T10:07:29.196739+0000","last_undegraded":"2024-09-21T10:07:29.196739+0000","last_fullsized":"2024-09-21T10:07:29.196739+0000","mapping_epoch":112,"log_start":"117'2500","ondisk_log_start":"117'2500","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":7340,"log_dups_size":0,"ondisk_log_size":7340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:20.723064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436600832,"num_objects":6662,"num_object_clones":0,"num_object_copies":19986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9840,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":114,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.480117+0000","last_change":"2024-09-21T10:03:53.842610+0000","last_active":"2024-09-21T10:07:23.480117+0000","last_peered":"2024-09-21T10:07:23.480117+0000","last_clean":"2024-09-21T10:07:23.480117+0000","last_became_active":"2024-09-21T10:03:53.841888+0000","last_became_peered":"2024-09-21T10:03:53.841888+0000","last_unstale":"2024-09-21T10:07:23.480117+0000","last_undegraded":"2024-09-21T10:07:23.480117+0000","last_fullsized":"2024-09-21T10:07:23.480117+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:05.833771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":677,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.501758+0000","last_change":"2024-09-21T10:02:46.665711+0000","last_active":"2024-09-21T10:07:23.501758+0000","last_peered":"2024-09-21T10:07:23.501758+0000","last_clean":"2024-09-21T10:07:23.501758+0000","last_became_active":"2024-09-21T10:02:46.665165+0000","last_became_peered":"2024-09-21T10:02:46.665165+0000","last_unstale":"2024-09-21T10:07:23.501758+0000","last_undegraded":"2024-09-21T10:07:23.501758+0000","last_fullsized":"2024-09-21T10:07:23.501758+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:08.886554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":111,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.480173+0000","last_change":"2024-09-21T10:03:53.838941+0000","last_active":"2024-09-21T10:07:23.480173+0000","last_peered":"2024-09-21T10:07:23.480173+0000","last_clean":"2024-09-21T10:07:23.480173+0000","last_became_active":"2024-09-21T10:03:53.838646+0000","last_became_peered":"2024-09-21T10:03:53.838646+0000","last_unstale":"2024-09-21T10:07:23.480173+0000","last_undegraded":"2024-09-21T10:07:23.480173+0000","last_fullsized":"2024-09-21T10:07:23.480173+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:25.167057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"130'9916","reported_seq":9964,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:26.638545+0000","last_change":"2024-09-21T10:05:19.210763+0000","last_active":"2024-09-21T10:07:26.638545+0000","last_peered":"2024-09-21T10:07:26.638545+0000","last_clean":"2024-09-21T10:07:26.638545+0000","last_became_active":"2024-09-21T10:05:19.210462+0000","last_became_peered":"2024-09-21T10:05:19.210462+0000","last_unstale":"2024-09-21T10:07:26.638545+0000","last_undegraded":"2024-09-21T10:07:26.638545+0000","last_fullsized":"2024-09-21T10:07:26.638545+0000","mapping_epoch":112,"log_start":"117'1000","ondisk_log_start":"117'1000","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8916,"log_dups_size":0,"ondisk_log_size":8916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:57.912861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443940864,"num_objects":6774,"num_object_clones":0,"num_object_copies":20322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9916,"num_write_kb":534080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":111,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.516972+0000","last_change":"2024-09-21T10:02:46.661363+0000","last_active":"2024-09-21T10:07:23.516972+0000","last_peered":"2024-09-21T10:07:23.516972+0000","last_clean":"2024-09-21T10:07:23.516972+0000","last_became_active":"2024-09-21T10:02:46.661005+0000","last_became_peered":"2024-09-21T10:02:46.661005+0000","last_unstale":"2024-09-21T10:07:23.516972+0000","last_undegraded":"2024-09-21T10:07:23.516972+0000","last_fullsized":"2024-09-21T10:07:23.516972+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:42.908471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":370,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.516657+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:07:09.516657+0000","last_peered":"2024-09-21T10:07:09.516657+0000","last_clean":"2024-09-21T10:07:09.516657+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:07:09.516657+0000","last_undegraded":"2024-09-21T10:07:09.516657+0000","last_fullsized":"2024-09-21T10:07:09.516657+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":79,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.505054+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:07:09.505054+0000","last_peered":"2024-09-21T10:07:09.505054+0000","last_clean":"2024-09-21T10:07:09.505054+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:07:09.505054+0000","last_undegraded":"2024-09-21T10:07:09.505054+0000","last_fullsized":"2024-09-21T10:07:09.505054+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504418+0000","last_change":"2024-09-21T10:02:46.667831+0000","last_active":"2024-09-21T10:07:09.504418+0000","last_peered":"2024-09-21T10:07:09.504418+0000","last_clean":"2024-09-21T10:07:09.504418+0000","last_became_active":"2024-09-21T10:02:46.667507+0000","last_became_peered":"2024-09-21T10:02:46.667507+0000","last_unstale":"2024-09-21T10:07:09.504418+0000","last_undegraded":"2024-09-21T10:07:09.504418+0000","last_fullsized":"2024-09-21T10:07:09.504418+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:23.990802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":102,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.506132+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:07:09.506132+0000","last_peered":"2024-09-21T10:07:09.506132+0000","last_clean":"2024-09-21T10:07:09.506132+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:07:09.506132+0000","last_undegraded":"2024-09-21T10:07:09.506132+0000","last_fullsized":"2024-09-21T10:07:09.506132+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.500935+0000","last_change":"2024-09-21T10:03:19.767147+0000","last_active":"2024-09-21T10:07:23.500935+0000","last_peered":"2024-09-21T10:07:23.500935+0000","last_clean":"2024-09-21T10:07:23.500935+0000","last_became_active":"2024-09-21T10:03:19.766822+0000","last_became_peered":"2024-09-21T10:03:19.766822+0000","last_unstale":"2024-09-21T10:07:23.500935+0000","last_undegraded":"2024-09-21T10:07:23.500935+0000","last_fullsized":"2024-09-21T10:07:23.500935+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:38.875912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":97,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:14.135817+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:07:14.135817+0000","last_peered":"2024-09-21T10:07:14.135817+0000","last_clean":"2024-09-21T10:07:14.135817+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:07:14.135817+0000","last_undegraded":"2024-09-21T10:07:14.135817+0000","last_fullsized":"2024-09-21T10:07:14.135817+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:19.302464+0000","last_change":"2024-09-21T10:02:46.659383+0000","last_active":"2024-09-21T10:07:19.302464+0000","last_peered":"2024-09-21T10:07:19.302464+0000","last_clean":"2024-09-21T10:07:19.302464+0000","last_became_active":"2024-09-21T10:02:46.659011+0000","last_became_peered":"2024-09-21T10:02:46.659011+0000","last_unstale":"2024-09-21T10:07:19.302464+0000","last_undegraded":"2024-09-21T10:07:19.302464+0000","last_fullsized":"2024-09-21T10:07:19.302464+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:04.650699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.482379+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:07:23.482379+0000","last_peered":"2024-09-21T10:07:23.482379+0000","last_clean":"2024-09-21T10:07:23.482379+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:07:23.482379+0000","last_undegraded":"2024-09-21T10:07:23.482379+0000","last_fullsized":"2024-09-21T10:07:23.482379+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.537129+0000","last_change":"2024-09-21T10:03:53.844673+0000","last_active":"2024-09-21T10:07:23.537129+0000","last_peered":"2024-09-21T10:07:23.537129+0000","last_clean":"2024-09-21T10:07:23.537129+0000","last_became_active":"2024-09-21T10:03:53.844313+0000","last_became_peered":"2024-09-21T10:03:53.844313+0000","last_unstale":"2024-09-21T10:07:23.537129+0000","last_undegraded":"2024-09-21T10:07:23.537129+0000","last_fullsized":"2024-09-21T10:07:23.537129+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:42.529301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":129,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.479871+0000","last_change":"2024-09-21T10:03:53.841417+0000","last_active":"2024-09-21T10:07:23.479871+0000","last_peered":"2024-09-21T10:07:23.479871+0000","last_clean":"2024-09-21T10:07:23.479871+0000","last_became_active":"2024-09-21T10:03:53.840993+0000","last_became_peered":"2024-09-21T10:03:53.840993+0000","last_unstale":"2024-09-21T10:07:23.479871+0000","last_undegraded":"2024-09-21T10:07:23.479871+0000","last_fullsized":"2024-09-21T10:07:23.479871+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:05.666490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":103,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.516650+0000","last_change":"2024-09-21T10:03:19.765682+0000","last_active":"2024-09-21T10:07:09.516650+0000","last_peered":"2024-09-21T10:07:09.516650+0000","last_clean":"2024-09-21T10:07:09.516650+0000","last_became_active":"2024-09-21T10:03:19.765344+0000","last_became_peered":"2024-09-21T10:03:19.765344+0000","last_unstale":"2024-09-21T10:07:09.516650+0000","last_undegraded":"2024-09-21T10:07:09.516650+0000","last_fullsized":"2024-09-21T10:07:09.516650+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:43.787280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":132,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.479996+0000","last_change":"2024-09-21T10:03:53.839372+0000","last_active":"2024-09-21T10:07:23.479996+0000","last_peered":"2024-09-21T10:07:23.479996+0000","last_clean":"2024-09-21T10:07:23.479996+0000","last_became_active":"2024-09-21T10:03:53.839053+0000","last_became_peered":"2024-09-21T10:03:53.839053+0000","last_unstale":"2024-09-21T10:07:23.479996+0000","last_undegraded":"2024-09-21T10:07:23.479996+0000","last_fullsized":"2024-09-21T10:07:23.479996+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:29.647982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":108,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.521920+0000","last_change":"2024-09-21T10:03:53.844946+0000","last_active":"2024-09-21T10:07:23.521920+0000","last_peered":"2024-09-21T10:07:23.521920+0000","last_clean":"2024-09-21T10:07:23.521920+0000","last_became_active":"2024-09-21T10:03:53.844613+0000","last_became_peered":"2024-09-21T10:03:53.844613+0000","last_unstale":"2024-09-21T10:07:23.521920+0000","last_undegraded":"2024-09-21T10:07:23.521920+0000","last_fullsized":"2024-09-21T10:07:23.521920+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:41.055573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.499434+0000","last_change":"2024-09-21T10:03:53.846948+0000","last_active":"2024-09-21T10:07:23.499434+0000","last_peered":"2024-09-21T10:07:23.499434+0000","last_clean":"2024-09-21T10:07:23.499434+0000","last_became_active":"2024-09-21T10:03:53.846614+0000","last_became_peered":"2024-09-21T10:03:53.846614+0000","last_unstale":"2024-09-21T10:07:23.499434+0000","last_undegraded":"2024-09-21T10:07:23.499434+0000","last_fullsized":"2024-09-21T10:07:23.499434+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:53.968372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":115,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.480853+0000","last_change":"2024-09-21T10:03:53.845201+0000","last_active":"2024-09-21T10:07:23.480853+0000","last_peered":"2024-09-21T10:07:23.480853+0000","last_clean":"2024-09-21T10:07:23.480853+0000","last_became_active":"2024-09-21T10:03:53.843822+0000","last_became_peered":"2024-09-21T10:03:53.843822+0000","last_unstale":"2024-09-21T10:07:23.480853+0000","last_undegraded":"2024-09-21T10:07:23.480853+0000","last_fullsized":"2024-09-21T10:07:23.480853+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:28.895373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":109,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504628+0000","last_change":"2024-09-21T10:03:19.768025+0000","last_active":"2024-09-21T10:07:09.504628+0000","last_peered":"2024-09-21T10:07:09.504628+0000","last_clean":"2024-09-21T10:07:09.504628+0000","last_became_active":"2024-09-21T10:03:19.767632+0000","last_became_peered":"2024-09-21T10:03:19.767632+0000","last_unstale":"2024-09-21T10:07:09.504628+0000","last_undegraded":"2024-09-21T10:07:09.504628+0000","last_fullsized":"2024-09-21T10:07:09.504628+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:40.023759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":135,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.499581+0000","last_change":"2024-09-21T10:03:53.840263+0000","last_active":"2024-09-21T10:07:23.499581+0000","last_peered":"2024-09-21T10:07:23.499581+0000","last_clean":"2024-09-21T10:07:23.499581+0000","last_became_active":"2024-09-21T10:03:53.839894+0000","last_became_peered":"2024-09-21T10:03:53.839894+0000","last_unstale":"2024-09-21T10:07:23.499581+0000","last_undegraded":"2024-09-21T10:07:23.499581+0000","last_fullsized":"2024-09-21T10:07:23.499581+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:52.489580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":100,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.517387+0000","last_change":"2024-09-21T10:02:12.846054+0000","last_active":"2024-09-21T10:07:23.517387+0000","last_peered":"2024-09-21T10:07:23.517387+0000","last_clean":"2024-09-21T10:07:23.517387+0000","last_became_active":"2024-09-21T10:02:12.843696+0000","last_became_peered":"2024-09-21T10:02:12.843696+0000","last_unstale":"2024-09-21T10:07:23.517387+0000","last_undegraded":"2024-09-21T10:07:23.517387+0000","last_fullsized":"2024-09-21T10:07:23.517387+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:32.280697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504326+0000","last_change":"2024-09-21T10:03:53.840992+0000","last_active":"2024-09-21T10:07:09.504326+0000","last_peered":"2024-09-21T10:07:09.504326+0000","last_clean":"2024-09-21T10:07:09.504326+0000","last_became_active":"2024-09-21T10:03:53.840551+0000","last_became_peered":"2024-09-21T10:03:53.840551+0000","last_unstale":"2024-09-21T10:07:09.504326+0000","last_undegraded":"2024-09-21T10:07:09.504326+0000","last_fullsized":"2024-09-21T10:07:09.504326+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:04.442126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":95,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.508147+0000","last_change":"2024-09-21T10:02:46.659513+0000","last_active":"2024-09-21T10:07:09.508147+0000","last_peered":"2024-09-21T10:07:09.508147+0000","last_clean":"2024-09-21T10:07:09.508147+0000","last_became_active":"2024-09-21T10:02:46.659130+0000","last_became_peered":"2024-09-21T10:02:46.659130+0000","last_unstale":"2024-09-21T10:07:09.508147+0000","last_undegraded":"2024-09-21T10:07:09.508147+0000","last_fullsized":"2024-09-21T10:07:09.508147+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:24.792175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":107,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.504552+0000","last_change":"2024-09-21T10:03:53.835006+0000","last_active":"2024-09-21T10:07:09.504552+0000","last_peered":"2024-09-21T10:07:09.504552+0000","last_clean":"2024-09-21T10:07:09.504552+0000","last_became_active":"2024-09-21T10:03:53.834608+0000","last_became_peered":"2024-09-21T10:03:53.834608+0000","last_unstale":"2024-09-21T10:07:09.504552+0000","last_undegraded":"2024-09-21T10:07:09.504552+0000","last_fullsized":"2024-09-21T10:07:09.504552+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:56.108606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":80,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:09.508391+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:07:09.508391+0000","last_peered":"2024-09-21T10:07:09.508391+0000","last_clean":"2024-09-21T10:07:09.508391+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:07:09.508391+0000","last_undegraded":"2024-09-21T10:07:09.508391+0000","last_fullsized":"2024-09-21T10:07:09.508391+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.482849+0000","last_change":"2024-09-21T10:03:53.844043+0000","last_active":"2024-09-21T10:07:23.482849+0000","last_peered":"2024-09-21T10:07:23.482849+0000","last_clean":"2024-09-21T10:07:23.482849+0000","last_became_active":"2024-09-21T10:03:53.842891+0000","last_became_peered":"2024-09-21T10:03:53.842891+0000","last_unstale":"2024-09-21T10:07:23.482849+0000","last_undegraded":"2024-09-21T10:07:23.482849+0000","last_fullsized":"2024-09-21T10:07:23.482849+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:39.891977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":123,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.480231+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:07:23.480231+0000","last_peered":"2024-09-21T10:07:23.480231+0000","last_clean":"2024-09-21T10:07:23.480231+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:07:23.480231+0000","last_undegraded":"2024-09-21T10:07:23.480231+0000","last_fullsized":"2024-09-21T10:07:23.480231+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2024-09-21T10:07:19.203098+0000","last_change":"2024-09-21T10:02:46.659818+0000","last_active":"2024-09-21T10:07:19.203098+0000","last_peered":"2024-09-21T10:07:19.203098+0000","last_clean":"2024-09-21T10:07:19.203098+0000","last_became_active":"2024-09-21T10:02:46.659457+0000","last_became_peered":"2024-09-21T10:02:46.659457+0000","last_unstale":"2024-09-21T10:07:19.203098+0000","last_undegraded":"2024-09-21T10:07:19.203098+0000","last_fullsized":"2024-09-21T10:07:19.203098+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:21.962330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":111,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.536916+0000","last_change":"2024-09-21T10:03:19.769549+0000","last_active":"2024-09-21T10:07:23.536916+0000","last_peered":"2024-09-21T10:07:23.536916+0000","last_clean":"2024-09-21T10:07:23.536916+0000","last_became_active":"2024-09-21T10:03:19.768854+0000","last_became_peered":"2024-09-21T10:03:19.768854+0000","last_unstale":"2024-09-21T10:07:23.536916+0000","last_undegraded":"2024-09-21T10:07:23.536916+0000","last_fullsized":"2024-09-21T10:07:23.536916+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:29.415039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":7042236440,"num_objects":107457,"num_object_clones":0,"num_object_copies":322371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107457,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159321,"num_write_kb":8536833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21107650560,"data_stored":21107638344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146021,"ondisk_log_size":146021,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234214,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2225592,"kb_used_data":2158320,"kb_used_omap":1,"kb_used_meta":67198,"kb_avail":91519560,"statfs":{"total":95995035648,"available":93716029440,"internally_reserved":0,"allocated":2210119680,"data_stored":2209427146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":68811211},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495852,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1780720,"kb_used_data":1709416,"kb_used_omap":1,"kb_used_meta":71294,"kb_avail":91964432,"statfs":{"total":95995035648,"available":94171578368,"internally_reserved":0,"allocated":1750441984,"data_stored":1749757822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":73005511},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757489,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3972264,"kb_used_data":3846816,"kb_used_omap":1,"kb_used_meta":125438,"kb_avail":89772888,"statfs":{"total":95995035648,"available":91927437312,"internally_reserved":0,"allocated":3939139584,"data_stored":3938458920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":128448966},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019125,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3525936,"kb_used_data":3421992,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":90219216,"statfs":{"total":95995035648,"available":92384477184,"internally_reserved":0,"allocated":3504119808,"data_stored":3503435159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":105773510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435309,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1368156,"kb_used_data":1318420,"kb_used_omap":5,"kb_used_meta":49722,"kb_avail":92376996,"statfs":{"total":95995035648,"available":94594043904,"internally_reserved":0,"allocated":1350062080,"data_stored":1349393111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":50915368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598836,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2701280,"kb_used_data":2607128,"kb_used_omap":7,"kb_used_meta":94136,"kb_avail":91043872,"statfs":{"total":95995035648,"available":93228924928,"internally_reserved":0,"allocated":2669699072,"data_stored":2669026249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":96395725},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762363,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2218904,"kb_used_data":2146960,"kb_used_omap":7,"kb_used_meta":71928,"kb_avail":91526248,"statfs":{"total":95995035648,"available":93722877952,"internally_reserved":0,"allocated":2198487040,"data_stored":2197821823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":73654525},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925889,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3605476,"kb_used_data":3497308,"kb_used_omap":7,"kb_used_meta":108024,"kb_avail":90139676,"statfs":{"total":95995035648,"available":92303028224,"internally_reserved":0,"allocated":3581243392,"data_stored":3580567289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":110616766},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3574005760,"data_stored":3574005760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2180382720,"data_stored":2180382720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2652573696,"data_stored":2652569624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1339949056,"data_stored":1339949056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3501195264,"data_stored":3501195264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3929214976,"data_stored":3929210904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1740177408,"data_stored":1740177408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2190151680,"data_stored":2190147608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:07:31.444 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:07:31.444 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:07:31.444 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-21T10:07:31.444 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:07:31.444 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:07:31.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-21T10:07:31.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.469 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:31.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:31.570 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:31.612 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:31.640 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:31 smithi045 ceph-mon[95976]: pgmap v379: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-09-21T10:07:31.640 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:31 smithi045 ceph-mon[95976]: from='client.34740 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:31.686 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:31 smithi032 ceph-mon[101299]: pgmap v379: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-09-21T10:07:31.686 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:31 smithi032 ceph-mon[101299]: from='client.34740 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:31.686 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:31 smithi032 ceph-mon[105748]: pgmap v379: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-09-21T10:07:31.687 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:31 smithi032 ceph-mon[105748]: from='client.34740 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:31.714 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:31.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:31.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:31.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:31.893 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:31.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:31.995 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:32.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:32.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:32.138 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:32.176 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:32.278 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:32.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.325 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:32.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:32.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:32.466 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:32.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:32.567 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:32.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.602 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:32.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:32.703 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:32.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:32.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:32.855 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:32.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.904 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:32.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:33.005 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.050 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:33.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:33.151 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:33.182 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:33.226 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:33.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:33.328 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:33.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.367 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:33.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:33.467 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:33.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:33.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:33.648 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:33.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:33.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:33 smithi032 ceph-mon[105748]: pgmap v380: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-09-21T10:07:33.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:33.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:33 smithi032 ceph-mon[101299]: pgmap v380: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-09-21T10:07:33.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:33.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:33.782 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:33.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:33.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:33 smithi045 ceph-mon[95976]: pgmap v380: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-09-21T10:07:33.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:33.883 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:33.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.937 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:33.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:33.964 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:07:33.964 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:07:34.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:34.073 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:34.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:34.226 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:34.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:34.327 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:34.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:34.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:34.458 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:34.486 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:34.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:34.588 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:34.634 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:34.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:34.693 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:07:34.735 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:34.796 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:34.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:34.898 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:34.927 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:34.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:35.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:35.059 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:35.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:35.142 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:07:35.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:35.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:35.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:35.297 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:35.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:35.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:35.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:35.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.476 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:35.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:35.564 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:07:35.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:35.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:35.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:35.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:35.735 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:35.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:35.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:35 smithi045 ceph-mon[95976]: pgmap v381: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-21T10:07:35.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:35 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3860460162' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:35.788 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:35 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:35.820 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:07:35.837 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:35.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.925 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:35.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:35 smithi032 ceph-mon[105748]: pgmap v381: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-21T10:07:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:35 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3860460162' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:35 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:35 smithi032 ceph-mon[101299]: pgmap v381: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-21T10:07:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:35 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3860460162' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:35 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:36.026 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:36.065 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:36.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:36.167 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:36.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:36.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:36.311 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:36.357 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:36.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:36.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:36.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:36.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:36.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:36.637 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:36.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:36.739 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:36.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.778 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:36.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:36.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:36 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:36.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:36 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:07:36.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:36 smithi045 ceph-mon[95976]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T10:07:36.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:36.914 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:36.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:36.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:36 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:36.925 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:36 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:07:36.925 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:36 smithi032 ceph-mon[105748]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T10:07:36.925 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:36 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:36.925 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:36 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:07:36.925 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:36 smithi032 ceph-mon[101299]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T10:07:37.016 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:37.059 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:37.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:37.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:37.197 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:37.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:37.268 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:07:37.268 INFO:teuthology.orchestra.run.smithi032.stdout: "success": "" 2024-09-21T10:07:37.268 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:07:37.298 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:37.339 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:37.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:37.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:37.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.473 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:37.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:37.574 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:37.618 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:37.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:37.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:37.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.763 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:37.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:37.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:37 smithi045 ceph-mon[95976]: pgmap v383: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-09-21T10:07:37.814 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:37 smithi032 ceph-mon[105748]: pgmap v383: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-09-21T10:07:37.815 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:37 smithi032 ceph-mon[101299]: pgmap v383: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-09-21T10:07:37.864 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:37.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:38.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:38.069 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:38.077 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:38.170 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:38.202 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:38.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:38.339 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:38.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:38.441 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:38.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.495 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:38.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:38.596 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:38.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:38.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:38.742 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:38.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.791 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:38.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:38.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:38.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.931 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:38.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:39.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:39.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.063 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:39.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:39.164 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:39.214 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:39.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:39.253 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:07:39.253 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:07:39.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:39.352 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:39.454 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:39.491 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:39.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:39.594 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:39.636 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:39.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:39.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:39 smithi032 ceph-mon[105748]: pgmap v384: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-21T10:07:39.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:39.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:39 smithi032 ceph-mon[101299]: pgmap v384: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-21T10:07:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:39.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:39.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:07:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:07:39] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.51.0" 2024-09-21T10:07:39.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:39.763 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:39.823 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:07:39.865 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:39.869 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:39 smithi045 ceph-mon[95976]: pgmap v384: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-21T10:07:39.869 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:39.869 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:39.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:39.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:40.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:40.059 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:40.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:40.161 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:40.194 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:40.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:40.296 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:40.302 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:07:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.325 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:40.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:40.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:40.458 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:40.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:40.560 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:40.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.625 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:40.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:40.678 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:07:40.726 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:40.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.768 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:40.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:40.869 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:40.914 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:40.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:41.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:41.042 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:41.144 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:41.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:41.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:41.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:41.311 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:41.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:41.412 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:41.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.446 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:41.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:41.547 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:41.581 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:41.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:41.683 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:41.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:41.819 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:41.857 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:41.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:41.878 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:41 smithi045 ceph-mon[95976]: pgmap v385: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-09-21T10:07:41.885 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:41 smithi032 ceph-mon[105748]: pgmap v385: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-09-21T10:07:41.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:41 smithi032 ceph-mon[101299]: pgmap v385: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-09-21T10:07:41.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:41.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.993 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:41.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:42.094 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:42.127 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:42.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:42.228 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:42.231 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:07:42.231 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:42.257 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:42.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:42.262 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:07:42.263 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:42.295 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:07:42.295 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:42.331 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:07:42.332 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:42.359 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:42.378 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:07:42.378 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:07:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.404 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:42.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:42.407 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:07:42.408 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:42.446 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:07:42.446 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:42.489 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:07:42.490 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:42.506 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:42.542 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:42.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:42.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:42.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.697 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:42.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:42.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:42 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2386332364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:42.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:42 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2386332364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:42 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2386332364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:42.798 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:42.830 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:42.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:42.931 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:42.966 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:42.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:43.068 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:43.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:43.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:43.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:43.236 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:43.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:171533 5.7 5:e50b112c:::benchmark_data_smithi032_140417_object38142:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:43.235781+0000", 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.025266398999999998, 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.025297852999999999, 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:43.264 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 171533 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.235781+0000", 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.235781+0000", 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.235780+0000", 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.235786+0000", 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2910000000000004e-06 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:43.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.235802+0000", 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5778e-05 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.235809+0000", 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0620000000000003e-06 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.245579+0000", 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0097698390000000006 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:43.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.245876+0000", 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000297275 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.245993+0000", 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011677500000000001 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.257754+0000", 2024-09-21T10:07:43.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011761077 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.257760+0000", 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6830000000000002e-06 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:171535 5.f 5:f7846680:::benchmark_data_smithi032_140417_object38144:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:43.242646+0000", 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.018401859999999999, 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018527971000000001, 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 171535 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.270 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.242646+0000", 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.242646+0000", 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.242644+0000", 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:43.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.242652+0000", 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3499999999999999e-06 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.242668+0000", 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6116999999999998e-05 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.242676+0000", 2024-09-21T10:07:43.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1049999999999995e-06 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.252664+0000", 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.009988027 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.252768+0000", 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000104542 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.252873+0000", 2024-09-21T10:07:43.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000104207 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:171542 5.f 5:fcbf707f:::benchmark_data_smithi032_140417_object38151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:43.250455+0000", 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.010592898999999999, 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.01079359, 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:43.274 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 171542 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.250455+0000", 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.250455+0000", 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:43.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.250453+0000", 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.250459+0000", 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.361e-06 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.250469+0000", 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0104e-05 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.250475+0000", 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5710000000000003e-06 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.259408+0000", 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0089329160000000008 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.259524+0000", 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000116035 2024-09-21T10:07:43.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:07:43.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.259638+0000", 2024-09-21T10:07:43.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011333200000000001 2024-09-21T10:07:43.278 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.278 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:43.278 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:171548 5.d e131/112 5:b211a51c:::benchmark_data_smithi032_140417_object38157:head v 131'10868, mlcod=131'10866)", 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:43.255171+0000", 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.005877195, 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0061512040000000004, 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.255171+0000", 2024-09-21T10:07:43.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.255171+0000", 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.255170+0000", 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.255177+0000", 2024-09-21T10:07:43.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9659999999999999e-06 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.255189+0000", 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2523e-05 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.255195+0000", 2024-09-21T10:07:43.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2960000000000004e-06 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.256164+0000", 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00096829399999999997 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.256186+0000", 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2694e-05 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.44543.0:171533 5.7 e131/112)", 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:43.258069+0000", 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0029782989999999998, 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0033171849999999998, 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.258069+0000", 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.258069+0000", 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.258069+0000", 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.258077+0000", 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4139999999999994e-06 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.258088+0000", 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1005000000000001e-05 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.258094+0000", 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1279999999999996e-06 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:171554 5.d e131/112)", 2024-09-21T10:07:43.285 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:43.259689+0000", 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.001358885, 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001743093, 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.259689+0000", 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.259689+0000", 2024-09-21T10:07:43.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.259688+0000", 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.259695+0000", 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6959999999999999e-06 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.259703+0000", 2024-09-21T10:07:43.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.653e-06 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:43.259709+0000", 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0290000000000002e-06 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 6 2024-09-21T10:07:43.288 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:07:43.338 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:43.374 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:43.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:43.476 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:43.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:43.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:43.539 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:07:43.539 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:07:43.539 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:07:43.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:07:43.613 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:43.668 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.673 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:43.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:43.773 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:43.815 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:43.866 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:43 smithi045 ceph-mon[95976]: pgmap v386: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2024-09-21T10:07:43.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:43 smithi032 ceph-mon[101299]: pgmap v386: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2024-09-21T10:07:43.876 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:43 smithi032 ceph-mon[105748]: pgmap v386: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2024-09-21T10:07:43.918 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:43.967 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:44.068 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:44.117 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:44.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:44.219 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:07:44.219 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:44.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.271 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:44.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:44.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:44.400 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:44.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:44.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:44.539 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:44.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:44.591 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:07:44.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:44.685 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:44.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:44.787 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:44.824 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:44.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:44.925 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:44.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:44.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:45.050 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:45.082 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:45.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:45.183 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.221 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:45.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:45.322 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:45.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:45.459 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:45.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.496 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:45.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:45.597 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:45.636 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:45.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:45.670 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:07:45.737 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:45.776 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:45.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:45 smithi045 ceph-mon[95976]: pgmap v387: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-21T10:07:45.879 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:45.920 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:45.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:45 smithi032 ceph-mon[105748]: pgmap v387: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-21T10:07:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:45 smithi032 ceph-mon[101299]: pgmap v387: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-21T10:07:46.022 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:46.067 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:46.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:46.168 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:46.208 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:46.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:46.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:46.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:46.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:46.479 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:46.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:46.581 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:46.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.622 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:46.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:46.723 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:46.747 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:46.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:46.849 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:46.894 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:46.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:46.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:47.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:47.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:47.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:47.154 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:47.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:47.256 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:47.284 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:47.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:47.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:47.416 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:07:47.417 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:07:47.417 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:07:47.417 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:07:47.417 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:07:47.422 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:47.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:47.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:47.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.557 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:47.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:47.658 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:47.690 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:47.780 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:47 smithi032 ceph-mon[105748]: pgmap v388: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-09-21T10:07:47.780 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:47 smithi032 ceph-mon[101299]: pgmap v388: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-09-21T10:07:47.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:47.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:47.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:47.949 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:47 smithi045 ceph-mon[95976]: pgmap v388: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-09-21T10:07:47.959 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:47.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:47.998 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:48.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:48.102 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:48.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:48.203 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:48.235 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:48.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:48.336 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:48.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.373 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:48.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:48.474 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:48.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:48.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:48.608 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:48.609 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:48.612 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:48.641 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:48.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:48.743 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:48.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:48.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:48.849 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:48.867 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:48.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:48.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:48.997 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.064 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:49.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:49.165 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:49.202 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:49.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:49.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:49.347 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:07:49.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.378 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:49.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:49.479 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:49.515 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:49.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:49.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:49 smithi045 ceph-mon[95976]: pgmap v389: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-09-21T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:49 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1607635314' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:49 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:49.636 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:07:49.653 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:49.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:49.681 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:07:49.755 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:49.788 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[101299]: pgmap v389: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-09-21T10:07:49.788 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:49.788 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:49.788 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1607635314' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:49.788 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:49.789 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:07:49.789 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:07:49.789 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:49.789 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:07:49] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.51.0" 2024-09-21T10:07:49.789 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[105748]: pgmap v389: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-09-21T10:07:49.789 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:49.789 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:49.789 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1607635314' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:49.790 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:49.790 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:07:49.790 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:07:49.790 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:07:49.801 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:49.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:49.940 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:49.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:50.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:50.073 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:50.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:50.174 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:50.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:50.303 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:50.341 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:50.443 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:50.479 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:50.581 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:50.617 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:50.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:50.718 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:50.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:50.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:50.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:50.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:50.916 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:50 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:07:50.916 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:50 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:07:50.916 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:50 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:50.916 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:50 smithi045 ceph-mon[95976]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T10:07:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:50 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:07:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:50 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:07:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:50 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:50 smithi032 ceph-mon[101299]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T10:07:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:50 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:07:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:50 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:07:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:50 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:50 smithi032 ceph-mon[105748]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T10:07:50.998 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:51.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:51.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:51.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:51.156 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:51.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:51.257 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:51.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:51.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:51.412 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:51.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:51.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:51.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:51.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.599 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:51.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:51.660 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:26455 5.f 5:fafd1498:::benchmark_data_smithi032_140417_object26454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:05:38.498668+0000", 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 133.157257128, 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.162765193, 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.661 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 26455 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498668+0000", 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498668+0000", 2024-09-21T10:07:51.662 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498666+0000", 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498721+0000", 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4707000000000003e-05 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.663 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498737+0000", 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5432000000000001e-05 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498744+0000", 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6490000000000008e-06 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498818+0000", 2024-09-21T10:07:51.666 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3506999999999999e-05 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498945+0000", 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012687899999999999 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.499099+0000", 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000154482 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.667 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.500191+0000", 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0010918830000000001 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.500356+0000", 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016515200000000001 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661326+0000", 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.160969892 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.668 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661393+0000", 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6958000000000005e-05 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661433+0000", 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0271000000000001e-05 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.669 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:87230 5.7 5:e1f8d9f0:::benchmark_data_smithi032_140417_object87229:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:06:15.869602+0000", 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 95.786323287000002, 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.106309551, 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 87230 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.670 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.869602+0000", 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.869602+0000", 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.869600+0000", 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.869629+0000", 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8875999999999999e-05 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.869641+0000", 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.165e-05 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.869648+0000", 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.703e-06 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.672 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.869920+0000", 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00027283100000000002 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.870025+0000", 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000104922 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.870143+0000", 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000117509 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.673 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873130+0000", 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029872029999999999 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.876744+0000", 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0036135770000000002 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.975845+0000", 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.099101784999999998 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.674 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.975885+0000", 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9137999999999997e-05 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.975911+0000", 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.6818e-05 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:87234 5.7 5:ef44b94e:::benchmark_data_smithi032_140417_object87233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:07:51.675 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:06:15.873481+0000", 2024-09-21T10:07:51.676 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 95.782444179999999, 2024-09-21T10:07:51.676 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.112517883, 2024-09-21T10:07:51.676 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.676 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.676 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.676 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.676 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 87234 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873481+0000", 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.677 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873481+0000", 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873479+0000", 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873508+0000", 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9122e-05 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.678 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873524+0000", 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6008e-05 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873532+0000", 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5320000000000001e-06 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873613+0000", 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1050000000000005e-05 2024-09-21T10:07:51.679 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873739+0000", 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012557700000000001 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.873887+0000", 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014836 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.876791+0000", 2024-09-21T10:07:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029036000000000001 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.876938+0000", 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000147482 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.985940+0000", 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.109002053 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.681 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.985974+0000", 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3599000000000001e-05 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:15.985999+0000", 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.512e-05 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:88545 5.7 5:e107905d:::benchmark_data_smithi032_140417_object88544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:06:16.997605+0000", 2024-09-21T10:07:51.682 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 94.658319974999998, 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.086335823000000006, 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 88545 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:16.997605+0000", 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.683 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:16.997605+0000", 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:16.997604+0000", 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:16.997632+0000", 2024-09-21T10:07:51.684 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7951000000000001e-05 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:16.997641+0000", 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5279999999999992e-06 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:16.997647+0000", 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8470000000000001e-06 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:17.000597+0000", 2024-09-21T10:07:51.685 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029498010000000002 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:17.000717+0000", 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000119533 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:17.000854+0000", 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013704099999999999 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.686 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:17.004981+0000", 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0041276769999999997 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:17.007619+0000", 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002637758 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:17.083846+0000", 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.076227125000000007 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.687 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:17.083900+0000", 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4209000000000002e-05 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:17.083941+0000", 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0635e-05 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:90448 5.7 5:ea19d755:::benchmark_data_smithi032_140417_object90447:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:07:51.688 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:06:18.726891+0000", 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 92.929034281, 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.079322302999999997, 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 90448 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.726891+0000", 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.689 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.726891+0000", 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.726889+0000", 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.726938+0000", 2024-09-21T10:07:51.690 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8387999999999998e-05 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.726950+0000", 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2292e-05 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.726958+0000", 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8329999999999994e-06 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.727003+0000", 2024-09-21T10:07:51.691 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5317e-05 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.727101+0000", 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8207000000000003e-05 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.727200+0000", 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8602000000000002e-05 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.729603+0000", 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024032239999999998 2024-09-21T10:07:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.729708+0000", 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010490800000000001 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.806152+0000", 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.076444162999999996 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.806188+0000", 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5765000000000001e-05 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.806213+0000", 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5244000000000001e-05 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:90691 5.7 5:ec9f4dac:::benchmark_data_smithi032_140417_object90690:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:06:18.994739+0000", 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 92.661185969000002, 2024-09-21T10:07:51.694 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.10392074, 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 90691 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.994739+0000", 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.695 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.994739+0000", 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.994737+0000", 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.994777+0000", 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0197000000000002e-05 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.994792+0000", 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4929000000000001e-05 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.994800+0000", 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8970000000000008e-06 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.994869+0000", 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8281999999999994e-05 2024-09-21T10:07:51.697 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.995026+0000", 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015719099999999999 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.995174+0000", 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014779600000000001 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.998723+0000", 2024-09-21T10:07:51.698 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0035497219999999999 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:18.999763+0000", 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001039246 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:19.098598+0000", 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.098835661000000005 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.699 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:19.098633+0000", 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5237999999999997e-05 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:19.098660+0000", 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.6509e-05 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:98426 5.7 5:e8e05447:::benchmark_data_smithi032_140417_object98425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:06:25.481550+0000", 2024-09-21T10:07:51.700 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 86.174375346999994, 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.078504905, 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 98426 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.481550+0000", 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.481550+0000", 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.481548+0000", 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.481589+0000", 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0902999999999998e-05 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.481602+0000", 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2945000000000001e-05 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.481608+0000", 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7010000000000001e-06 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.481657+0000", 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9098999999999999e-05 2024-09-21T10:07:51.703 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.481754+0000", 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.7235e-05 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.481841+0000", 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7070000000000005e-05 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.484566+0000", 2024-09-21T10:07:51.704 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0027246380000000001 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.484659+0000", 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3728999999999997e-05 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.559925+0000", 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.075265896999999998 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.560019+0000", 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3858999999999997e-05 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:25.560055+0000", 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5667999999999998e-05 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:99168 5.7 5:e036d327:::benchmark_data_smithi032_140417_object99167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:06:26.270208+0000", 2024-09-21T10:07:51.706 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 85.385716861999995, 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14999678999999999, 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 99168 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.270208+0000", 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.270208+0000", 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.270207+0000", 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.270248+0000", 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0963000000000001e-05 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.270263+0000", 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5356e-05 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.270269+0000", 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2729999999999999e-06 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.270402+0000", 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013229800000000001 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.709 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.270540+0000", 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000138515 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.270674+0000", 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013433999999999999 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.277010+0000", 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0063353569999999998 2024-09-21T10:07:51.710 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.277017+0000", 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0979999999999996e-06 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.420065+0000", 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.143048593 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.711 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.420168+0000", 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000102069 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.420205+0000", 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7555999999999998e-05 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:99172 5.7 5:efc380b5:::benchmark_data_smithi032_140417_object99171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:06:26.272002+0000", 2024-09-21T10:07:51.712 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 85.383923491999994, 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.149814949, 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 99172 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.272002+0000", 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.713 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.272002+0000", 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.272001+0000", 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.272072+0000", 2024-09-21T10:07:51.714 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1161999999999994e-05 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.272086+0000", 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4673e-05 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.272094+0000", 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2439999999999996e-06 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.715 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.273815+0000", 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0017213420000000001 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.273957+0000", 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014161600000000001 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.274110+0000", 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015339300000000001 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.717 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:51.718 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.718 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.280292+0000", 2024-09-21T10:07:51.718 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0061820520000000004 2024-09-21T10:07:51.718 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.718 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.718 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.280308+0000", 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5994e-05 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.421733+0000", 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14142484399999999 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.421780+0000", 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7219999999999999e-05 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.421817+0000", 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6541999999999997e-05 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:99176 5.7 5:e8c31278:::benchmark_data_smithi032_140417_object99175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:06:26.275441+0000", 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 85.380484236000001, 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.150418629, 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.720 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 99176 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.275441+0000", 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.721 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.275441+0000", 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.275439+0000", 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.275541+0000", 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010180599999999999 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.722 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.275556+0000", 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5033e-05 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.275564+0000", 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2810000000000003e-06 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.277042+0000", 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0014786370000000001 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.723 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.277167+0000", 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000124998 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.277288+0000", 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120821 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.280500+0000", 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0032119800000000001 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.724 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.282044+0000", 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015436130000000001 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.425666+0000", 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.143622164 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.425820+0000", 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015454499999999999 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.725 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:06:26.425860+0000", 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9273000000000002e-05 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:174096 5.7 5:ec31da91:::benchmark_data_smithi032_140417_object40705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:46.428410+0000", 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 5.227515124, 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.10797308, 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.726 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174096 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.428410+0000", 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.428410+0000", 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.727 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.428409+0000", 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.428415+0000", 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3289999999999999e-06 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.428427+0000", 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1854e-05 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.728 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.428435+0000", 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1689999999999992e-06 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.428608+0000", 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017321799999999999 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.428747+0000", 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000138627 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.729 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.428848+0000", 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000101578 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.434652+0000", 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0058032739999999998 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.434697+0000", 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5039e-05 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.730 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.536314+0000", 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.101617814 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.536349+0000", 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4020000000000003e-05 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:46.536383+0000", 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4595999999999999e-05 2024-09-21T10:07:51.731 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:174820 5.b e131/112 5:d02b3370:::benchmark_data_smithi032_140417_object41429:head v 131'10897, mlcod=131'10895)", 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:47.454746+0000", 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.2011789569999998, 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.080534015, 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.454746+0000", 2024-09-21T10:07:51.732 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.454746+0000", 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.454745+0000", 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.454752+0000", 2024-09-21T10:07:51.733 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3600000000000001e-06 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.454762+0000", 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0546e-05 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.454768+0000", 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.173e-06 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.734 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.454820+0000", 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1665000000000001e-05 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.454840+0000", 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0288e-05 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535242+0000", 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.080401405999999995 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.735 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535280+0000", 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8433999999999998e-05 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:174826 5.e e131/112 5:74ee85e6:::benchmark_data_smithi032_140417_object41435:head v 131'10955, mlcod=131'10952)", 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:47.458683+0000", 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.1972418039999999, 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.076581098, 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.736 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.458683+0000", 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.458683+0000", 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.458683+0000", 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:07:51.737 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.458689+0000", 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0689999999999998e-06 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.458699+0000", 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0268000000000001e-05 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.458705+0000", 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3629999999999999e-06 2024-09-21T10:07:51.738 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462005+0000", 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003300037 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462024+0000", 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8663000000000001e-05 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535225+0000", 2024-09-21T10:07:51.739 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.073201047000000005 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535264+0000", 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9334e-05 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:174822 5.3 e131/112 5:c22d371c:::benchmark_data_smithi032_140417_object41431:head v 131'10968, mlcod=131'10966)", 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:47.459220+0000", 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.1967048020000002, 2024-09-21T10:07:51.740 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.083203977999999998, 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459220+0000", 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459220+0000", 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.741 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459220+0000", 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459225+0000", 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.631e-06 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459235+0000", 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8789999999999998e-06 2024-09-21T10:07:51.742 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459240+0000", 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2480000000000004e-06 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459288+0000", 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7691000000000001e-05 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459306+0000", 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7901e-05 2024-09-21T10:07:51.743 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.542390+0000", 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.083084323000000002 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.542424+0000", 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4056000000000001e-05 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.744 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:174828 5.4 e131/112 5:2dbed2fe:::benchmark_data_smithi032_140417_object41437:head v 131'11070, mlcod=131'11068)", 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:47.459447+0000", 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.1964782620000003, 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.085842349999999998, 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459447+0000", 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459447+0000", 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459446+0000", 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459452+0000", 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2890000000000003e-06 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.746 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459463+0000", 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0569e-05 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459469+0000", 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0499999999999997e-06 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459533+0000", 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4170000000000004e-05 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.747 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.459552+0000", 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8561000000000001e-05 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.545262+0000", 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.085710626999999998 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.545289+0000", 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.6818e-05 2024-09-21T10:07:51.748 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:174821 5.b e131/112 5:de11bd24:::benchmark_data_smithi032_140417_object41430:head v 131'10898, mlcod=131'10895)", 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:47.462106+0000", 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.193819381, 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.080344670000000007, 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.749 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462106+0000", 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462106+0000", 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462105+0000", 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.750 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462112+0000", 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8040000000000004e-06 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462122+0000", 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0071000000000001e-05 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462128+0000", 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6599999999999998e-06 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.751 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462176+0000", 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7806999999999998e-05 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.462193+0000", 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6699000000000001e-05 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.542427+0000", 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.080233840000000001 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.542450+0000", 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3598999999999999e-05 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:174829 5.f 5:f2a4288f:::benchmark_data_smithi032_140417_object41438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:47.463366+0000", 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.1925586560000001, 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.077879657000000005, 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.753 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174829 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.463366+0000", 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.463366+0000", 2024-09-21T10:07:51.754 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.463365+0000", 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.463371+0000", 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2899999999999999e-06 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.463391+0000", 2024-09-21T10:07:51.755 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9839000000000001e-05 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.463398+0000", 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8560000000000001e-06 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.463571+0000", 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017304399999999999 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.756 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.463670+0000", 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.9259999999999995e-05 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.463776+0000", 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000105418 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535218+0000", 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.071441905 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.757 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.540679+0000", 2024-09-21T10:07:51.758 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0054614260000000001 2024-09-21T10:07:51.758 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.758 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.758 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.758 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.541134+0000", 2024-09-21T10:07:51.759 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00045522200000000003 2024-09-21T10:07:51.759 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.541209+0000", 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5042000000000004e-05 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.541246+0000", 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6699999999999998e-05 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.760 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:174833 5.f 5:f7443eda:::benchmark_data_smithi032_140417_object41442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:47.465142+0000", 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.1907829940000001, 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.090380094999999994, 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174833 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.761 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465142+0000", 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465142+0000", 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465141+0000", 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:07:51.762 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465147+0000", 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6699999999999999e-06 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465157+0000", 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0711e-05 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465164+0000", 2024-09-21T10:07:51.763 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7229999999999998e-06 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535427+0000", 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.070263307999999997 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535568+0000", 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014104699999999999 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535702+0000", 2024-09-21T10:07:51.764 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013337000000000001 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.554257+0000", 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018555426 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.554296+0000", 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9029000000000002e-05 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:07:51.765 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.555442+0000", 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0011455289999999999 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.555482+0000", 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0729999999999998e-05 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.555522+0000", 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9745e-05 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.766 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:174832 5.2 e131/112 5:47cdcc22:::benchmark_data_smithi032_140417_object41441:head v 131'10859, mlcod=131'10856)", 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:47.465775+0000", 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.1901496470000001, 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.084530684999999994, 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465775+0000", 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.767 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465775+0000", 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465775+0000", 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465780+0000", 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4149999999999998e-06 2024-09-21T10:07:51.768 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465789+0000", 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9779999999999994e-06 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465795+0000", 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8220000000000003e-06 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.465847+0000", 2024-09-21T10:07:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2095000000000002e-05 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.534966+0000", 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.069118880999999993 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.550268+0000", 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015301989 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.770 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.550306+0000", 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8074000000000002e-05 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:174835 5.3 e131/112 5:c36e94df:::benchmark_data_smithi032_140417_object41444:head v 131'10969, mlcod=131'10966)", 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:07:47.469199+0000", 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.1867258270000001, 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.093775549999999999, 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:07:51.771 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.469199+0000", 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.469199+0000", 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.469198+0000", 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.469206+0000", 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3069999999999997e-06 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535253+0000", 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.066047874000000006 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535263+0000", 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.1789999999999997e-06 2024-09-21T10:07:51.773 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535307+0000", 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4335999999999997e-05 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.535328+0000", 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1015000000000001e-05 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.562940+0000", 2024-09-21T10:07:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.027612040000000001 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:07:47.562975+0000", 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4864999999999999e-05 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:07:51.775 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:07:51.776 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:51 smithi045 ceph-mon[95976]: pgmap v391: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-21T10:07:51.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:51.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:51.878 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:51.912 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:51.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:51 smithi032 ceph-mon[101299]: pgmap v391: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-21T10:07:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:51 smithi032 ceph-mon[105748]: pgmap v391: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-21T10:07:52.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.056 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:52.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:52.157 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:52.186 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:52.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:52.288 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:52.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:52.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:52.435 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:52.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.479 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:52.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:52.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:52.625 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:52.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:52.628 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:07:52.726 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:52.762 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:52.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:52.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:52.905 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:52.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:53.007 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:53.048 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:07:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.056 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:53.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:53.157 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:53.199 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:53.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:53.300 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:53.332 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:07:53.332 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:53.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:53.362 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:07:53.363 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:07:53.378 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:07:53.378 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:53.418 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:07:53.418 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:53.447 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:53.453 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:07:53.453 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:53.484 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:53.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:53.486 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:07:53.486 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:07:53.520 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:07:53.520 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:53.554 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:07:53.554 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:53.585 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:53.588 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:07:53.589 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:53.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.630 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:53.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:53.731 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.764 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:53.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:53.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:07:53.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:53 smithi045 ceph-mon[95976]: pgmap v392: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-09-21T10:07:53.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:53 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2549238052' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:53.865 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.899 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:53.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:53 smithi032 ceph-mon[105748]: pgmap v392: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-09-21T10:07:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:53 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2549238052' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:53 smithi032 ceph-mon[101299]: pgmap v392: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-09-21T10:07:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:53 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2549238052' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:54.000 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:54.048 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:54.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:54.149 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:54.183 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:54.285 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:54.320 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2024-09-21T10:07:52.192743+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5936322557,"num_objects":90537,"num_object_clones":0,"num_object_copies":271611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90537,"num_whiteouts":0,"num_read":6391,"num_read_kb":13846,"num_write":177392,"num_write_kb":8551874,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147951,"ondisk_log_size":147951,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18223032,"kb_used_data":17431664,"kb_used_omap":35,"kb_used_meta":791068,"kb_avail":731738184,"statfs":{"total":767960285184,"available":749299900416,"internally_reserved":0,"allocated":17850023936,"data_stored":17844558215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":810054350},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,3,7,12,6,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-525402112,"num_objects":-8017,"num_object_clones":0,"num_object_copies":-24051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8017,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002706"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":120,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.629846+0000","last_change":"2024-09-21T10:03:53.843548+0000","last_active":"2024-09-21T10:07:49.629846+0000","last_peered":"2024-09-21T10:07:49.629846+0000","last_clean":"2024-09-21T10:07:49.629846+0000","last_became_active":"2024-09-21T10:03:53.843182+0000","last_became_peered":"2024-09-21T10:03:53.843182+0000","last_unstale":"2024-09-21T10:07:49.629846+0000","last_undegraded":"2024-09-21T10:07:49.629846+0000","last_fullsized":"2024-09-21T10:07:49.629846+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:09.464209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.503195+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:07:23.503195+0000","last_peered":"2024-09-21T10:07:23.503195+0000","last_clean":"2024-09-21T10:07:23.503195+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:07:23.503195+0000","last_undegraded":"2024-09-21T10:07:23.503195+0000","last_fullsized":"2024-09-21T10:07:23.503195+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":94,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.500873+0000","last_change":"2024-09-21T10:03:53.839987+0000","last_active":"2024-09-21T10:07:23.500873+0000","last_peered":"2024-09-21T10:07:23.500873+0000","last_clean":"2024-09-21T10:07:23.500873+0000","last_became_active":"2024-09-21T10:03:53.839664+0000","last_became_peered":"2024-09-21T10:03:53.839664+0000","last_unstale":"2024-09-21T10:07:23.500873+0000","last_undegraded":"2024-09-21T10:07:23.500873+0000","last_fullsized":"2024-09-21T10:07:23.500873+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:51.726392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.648950+0000","last_change":"2024-09-21T10:03:53.837496+0000","last_active":"2024-09-21T10:07:49.648950+0000","last_peered":"2024-09-21T10:07:49.648950+0000","last_clean":"2024-09-21T10:07:49.648950+0000","last_became_active":"2024-09-21T10:03:53.837166+0000","last_became_peered":"2024-09-21T10:03:53.837166+0000","last_unstale":"2024-09-21T10:07:49.648950+0000","last_undegraded":"2024-09-21T10:07:49.648950+0000","last_fullsized":"2024-09-21T10:07:49.648950+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:45.795571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.643064+0000","last_change":"2024-09-21T10:03:53.844990+0000","last_active":"2024-09-21T10:07:49.643064+0000","last_peered":"2024-09-21T10:07:49.643064+0000","last_clean":"2024-09-21T10:07:49.643064+0000","last_became_active":"2024-09-21T10:03:53.844664+0000","last_became_peered":"2024-09-21T10:03:53.844664+0000","last_unstale":"2024-09-21T10:07:49.643064+0000","last_undegraded":"2024-09-21T10:07:49.643064+0000","last_fullsized":"2024-09-21T10:07:49.643064+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:01.031522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":80,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.503189+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:07:23.503189+0000","last_peered":"2024-09-21T10:07:23.503189+0000","last_clean":"2024-09-21T10:07:23.503189+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:07:23.503189+0000","last_undegraded":"2024-09-21T10:07:23.503189+0000","last_fullsized":"2024-09-21T10:07:23.503189+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":120,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.630321+0000","last_change":"2024-09-21T10:03:53.845785+0000","last_active":"2024-09-21T10:07:49.630321+0000","last_peered":"2024-09-21T10:07:49.630321+0000","last_clean":"2024-09-21T10:07:49.630321+0000","last_became_active":"2024-09-21T10:03:53.845303+0000","last_became_peered":"2024-09-21T10:03:53.845303+0000","last_unstale":"2024-09-21T10:07:49.630321+0000","last_undegraded":"2024-09-21T10:07:49.630321+0000","last_fullsized":"2024-09-21T10:07:49.630321+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:04.976007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":103,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.505527+0000","last_change":"2024-09-21T10:02:12.838786+0000","last_active":"2024-09-21T10:07:23.505527+0000","last_peered":"2024-09-21T10:07:23.505527+0000","last_clean":"2024-09-21T10:07:23.505527+0000","last_became_active":"2024-09-21T10:02:12.838242+0000","last_became_peered":"2024-09-21T10:02:12.838242+0000","last_unstale":"2024-09-21T10:07:23.505527+0000","last_undegraded":"2024-09-21T10:07:23.505527+0000","last_fullsized":"2024-09-21T10:07:23.505527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:31.071136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"131'10983","reported_seq":11012,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.076540+0000","last_change":"2024-09-21T10:05:19.211627+0000","last_active":"2024-09-21T10:07:49.076540+0000","last_peered":"2024-09-21T10:07:49.076540+0000","last_clean":"2024-09-21T10:07:49.076540+0000","last_became_active":"2024-09-21T10:05:19.211261+0000","last_became_peered":"2024-09-21T10:05:19.211261+0000","last_unstale":"2024-09-21T10:07:49.076540+0000","last_undegraded":"2024-09-21T10:07:49.076540+0000","last_fullsized":"2024-09-21T10:07:49.076540+0000","mapping_epoch":112,"log_start":"117'2400","ondisk_log_start":"117'2400","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8583,"log_dups_size":0,"ondisk_log_size":8583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:10.360788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374145024,"num_objects":5709,"num_object_clones":0,"num_object_copies":17127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10983,"num_write_kb":534144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.503634+0000","last_change":"2024-09-21T10:02:12.840334+0000","last_active":"2024-09-21T10:07:23.503634+0000","last_peered":"2024-09-21T10:07:23.503634+0000","last_clean":"2024-09-21T10:07:23.503634+0000","last_became_active":"2024-09-21T10:02:12.839928+0000","last_became_peered":"2024-09-21T10:02:12.839928+0000","last_unstale":"2024-09-21T10:07:23.503634+0000","last_undegraded":"2024-09-21T10:07:23.503634+0000","last_fullsized":"2024-09-21T10:07:23.503634+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:38.593644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":80,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.561570+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T10:07:35.561570+0000","last_peered":"2024-09-21T10:07:35.561570+0000","last_clean":"2024-09-21T10:07:35.561570+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T10:07:35.561570+0000","last_undegraded":"2024-09-21T10:07:35.561570+0000","last_fullsized":"2024-09-21T10:07:35.561570+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"131'10928","reported_seq":10978,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:48.123783+0000","last_change":"2024-09-21T10:05:19.213525+0000","last_active":"2024-09-21T10:07:48.123783+0000","last_peered":"2024-09-21T10:07:48.123783+0000","last_clean":"2024-09-21T10:07:48.123783+0000","last_became_active":"2024-09-21T10:05:19.212434+0000","last_became_peered":"2024-09-21T10:05:19.212434+0000","last_unstale":"2024-09-21T10:07:48.123783+0000","last_undegraded":"2024-09-21T10:07:48.123783+0000","last_fullsized":"2024-09-21T10:07:48.123783+0000","mapping_epoch":112,"log_start":"117'900","ondisk_log_start":"117'900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10028,"log_dups_size":0,"ondisk_log_size":10028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:35.483451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370933760,"num_objects":5660,"num_object_clones":0,"num_object_copies":16980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10928,"num_write_kb":530816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.503978+0000","last_change":"2024-09-21T10:02:46.662169+0000","last_active":"2024-09-21T10:07:23.503978+0000","last_peered":"2024-09-21T10:07:23.503978+0000","last_clean":"2024-09-21T10:07:23.503978+0000","last_became_active":"2024-09-21T10:02:46.661721+0000","last_became_peered":"2024-09-21T10:02:46.661721+0000","last_unstale":"2024-09-21T10:07:23.503978+0000","last_undegraded":"2024-09-21T10:07:23.503978+0000","last_fullsized":"2024-09-21T10:07:23.503978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:10.414152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":110,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.566333+0000","last_change":"2024-09-21T10:03:53.840738+0000","last_active":"2024-09-21T10:07:35.566333+0000","last_peered":"2024-09-21T10:07:35.566333+0000","last_clean":"2024-09-21T10:07:35.566333+0000","last_became_active":"2024-09-21T10:03:53.840363+0000","last_became_peered":"2024-09-21T10:03:53.840363+0000","last_unstale":"2024-09-21T10:07:35.566333+0000","last_undegraded":"2024-09-21T10:07:35.566333+0000","last_fullsized":"2024-09-21T10:07:35.566333+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:26.371181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"131'10924","reported_seq":10974,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:48.124052+0000","last_change":"2024-09-21T10:05:19.213181+0000","last_active":"2024-09-21T10:07:48.124052+0000","last_peered":"2024-09-21T10:07:48.124052+0000","last_clean":"2024-09-21T10:07:48.124052+0000","last_became_active":"2024-09-21T10:05:19.212895+0000","last_became_peered":"2024-09-21T10:05:19.212895+0000","last_unstale":"2024-09-21T10:07:48.124052+0000","last_undegraded":"2024-09-21T10:07:48.124052+0000","last_fullsized":"2024-09-21T10:07:48.124052+0000","mapping_epoch":112,"log_start":"117'900","ondisk_log_start":"117'900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10024,"log_dups_size":0,"ondisk_log_size":10024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:40:31.162520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377225216,"num_objects":5756,"num_object_clones":0,"num_object_copies":17268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10924,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":81,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.503634+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T10:07:23.503634+0000","last_peered":"2024-09-21T10:07:23.503634+0000","last_clean":"2024-09-21T10:07:23.503634+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T10:07:23.503634+0000","last_undegraded":"2024-09-21T10:07:23.503634+0000","last_fullsized":"2024-09-21T10:07:23.503634+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":95,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.578001+0000","last_change":"2024-09-21T10:02:46.663127+0000","last_active":"2024-09-21T10:07:35.578001+0000","last_peered":"2024-09-21T10:07:35.578001+0000","last_clean":"2024-09-21T10:07:35.578001+0000","last_became_active":"2024-09-21T10:02:46.662791+0000","last_became_peered":"2024-09-21T10:02:46.662791+0000","last_unstale":"2024-09-21T10:07:35.578001+0000","last_undegraded":"2024-09-21T10:07:35.578001+0000","last_fullsized":"2024-09-21T10:07:35.578001+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:59.990122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"131'10847","reported_seq":10896,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:48.140088+0000","last_change":"2024-09-21T10:05:19.212279+0000","last_active":"2024-09-21T10:07:48.140088+0000","last_peered":"2024-09-21T10:07:48.140088+0000","last_clean":"2024-09-21T10:07:48.140088+0000","last_became_active":"2024-09-21T10:05:19.211960+0000","last_became_peered":"2024-09-21T10:05:19.211960+0000","last_unstale":"2024-09-21T10:07:48.140088+0000","last_undegraded":"2024-09-21T10:07:48.140088+0000","last_fullsized":"2024-09-21T10:07:48.140088+0000","mapping_epoch":112,"log_start":"117'800","ondisk_log_start":"117'800","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10047,"log_dups_size":0,"ondisk_log_size":10047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:40.251441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383188992,"num_objects":5847,"num_object_clones":0,"num_object_copies":17541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10847,"num_write_kb":534208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.503474+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:07:23.503474+0000","last_peered":"2024-09-21T10:07:23.503474+0000","last_clean":"2024-09-21T10:07:23.503474+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:07:23.503474+0000","last_undegraded":"2024-09-21T10:07:23.503474+0000","last_fullsized":"2024-09-21T10:07:23.503474+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.643463+0000","last_change":"2024-09-21T10:02:46.662651+0000","last_active":"2024-09-21T10:07:49.643463+0000","last_peered":"2024-09-21T10:07:49.643463+0000","last_clean":"2024-09-21T10:07:49.643463+0000","last_became_active":"2024-09-21T10:02:46.662304+0000","last_became_peered":"2024-09-21T10:02:46.662304+0000","last_unstale":"2024-09-21T10:07:49.643463+0000","last_undegraded":"2024-09-21T10:07:49.643463+0000","last_fullsized":"2024-09-21T10:07:49.643463+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:06.746929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"131'11098","reported_seq":11149,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:48.080999+0000","last_change":"2024-09-21T10:05:19.211133+0000","last_active":"2024-09-21T10:07:48.080999+0000","last_peered":"2024-09-21T10:07:48.080999+0000","last_clean":"2024-09-21T10:07:48.080999+0000","last_became_active":"2024-09-21T10:05:19.210822+0000","last_became_peered":"2024-09-21T10:05:19.210822+0000","last_unstale":"2024-09-21T10:07:48.080999+0000","last_undegraded":"2024-09-21T10:07:48.080999+0000","last_fullsized":"2024-09-21T10:07:48.080999+0000","mapping_epoch":112,"log_start":"117'1000","ondisk_log_start":"117'1000","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10098,"log_dups_size":0,"ondisk_log_size":10098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:02.399086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380305432,"num_objects":5804,"num_object_clones":0,"num_object_copies":17412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5804,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11098,"num_write_kb":540801,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":118,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.629681+0000","last_change":"2024-09-21T10:03:53.840511+0000","last_active":"2024-09-21T10:07:49.629681+0000","last_peered":"2024-09-21T10:07:49.629681+0000","last_clean":"2024-09-21T10:07:49.629681+0000","last_became_active":"2024-09-21T10:03:53.840136+0000","last_became_peered":"2024-09-21T10:03:53.840136+0000","last_unstale":"2024-09-21T10:07:49.629681+0000","last_undegraded":"2024-09-21T10:07:49.629681+0000","last_fullsized":"2024-09-21T10:07:49.629681+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:24.765990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":94,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.501914+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:07:23.501914+0000","last_peered":"2024-09-21T10:07:23.501914+0000","last_clean":"2024-09-21T10:07:23.501914+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:07:23.501914+0000","last_undegraded":"2024-09-21T10:07:23.501914+0000","last_fullsized":"2024-09-21T10:07:23.501914+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"131'10941","reported_seq":10991,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.511604+0000","last_change":"2024-09-21T10:05:19.214310+0000","last_active":"2024-09-21T10:07:49.511604+0000","last_peered":"2024-09-21T10:07:49.511604+0000","last_clean":"2024-09-21T10:07:49.511604+0000","last_became_active":"2024-09-21T10:05:19.213979+0000","last_became_peered":"2024-09-21T10:05:19.213979+0000","last_unstale":"2024-09-21T10:07:49.511604+0000","last_undegraded":"2024-09-21T10:07:49.511604+0000","last_fullsized":"2024-09-21T10:07:49.511604+0000","mapping_epoch":112,"log_start":"118'2800","ondisk_log_start":"118'2800","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8141,"log_dups_size":0,"ondisk_log_size":8141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:05.291457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370999296,"num_objects":5661,"num_object_clones":0,"num_object_copies":16983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10941,"num_write_kb":531264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.578051+0000","last_change":"2024-09-21T10:02:46.663900+0000","last_active":"2024-09-21T10:07:35.578051+0000","last_peered":"2024-09-21T10:07:35.578051+0000","last_clean":"2024-09-21T10:07:35.578051+0000","last_became_active":"2024-09-21T10:02:46.663594+0000","last_became_peered":"2024-09-21T10:02:46.663594+0000","last_unstale":"2024-09-21T10:07:35.578051+0000","last_undegraded":"2024-09-21T10:07:35.578051+0000","last_fullsized":"2024-09-21T10:07:35.578051+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:24.140061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":109,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.501583+0000","last_change":"2024-09-21T10:03:19.765623+0000","last_active":"2024-09-21T10:07:23.501583+0000","last_peered":"2024-09-21T10:07:23.501583+0000","last_clean":"2024-09-21T10:07:23.501583+0000","last_became_active":"2024-09-21T10:03:19.765227+0000","last_became_peered":"2024-09-21T10:03:19.765227+0000","last_unstale":"2024-09-21T10:07:23.501583+0000","last_undegraded":"2024-09-21T10:07:23.501583+0000","last_fullsized":"2024-09-21T10:07:23.501583+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:52.227826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"131'11108","reported_seq":11137,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.399800+0000","last_change":"2024-09-21T10:05:19.212082+0000","last_active":"2024-09-21T10:07:49.399800+0000","last_peered":"2024-09-21T10:07:49.399800+0000","last_clean":"2024-09-21T10:07:49.399800+0000","last_became_active":"2024-09-21T10:05:19.211673+0000","last_became_peered":"2024-09-21T10:05:19.211673+0000","last_unstale":"2024-09-21T10:07:49.399800+0000","last_undegraded":"2024-09-21T10:07:49.399800+0000","last_fullsized":"2024-09-21T10:07:49.399800+0000","mapping_epoch":112,"log_start":"117'1100","ondisk_log_start":"117'1100","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:12.775587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372768768,"num_objects":5688,"num_object_clones":0,"num_object_copies":17064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11108,"num_write_kb":537472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.566795+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:07:35.566795+0000","last_peered":"2024-09-21T10:07:35.566795+0000","last_clean":"2024-09-21T10:07:35.566795+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:07:35.566795+0000","last_undegraded":"2024-09-21T10:07:35.566795+0000","last_fullsized":"2024-09-21T10:07:35.566795+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":94,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.501743+0000","last_change":"2024-09-21T10:02:46.664014+0000","last_active":"2024-09-21T10:07:23.501743+0000","last_peered":"2024-09-21T10:07:23.501743+0000","last_clean":"2024-09-21T10:07:23.501743+0000","last_became_active":"2024-09-21T10:02:46.663640+0000","last_became_peered":"2024-09-21T10:02:46.663640+0000","last_unstale":"2024-09-21T10:07:23.501743+0000","last_undegraded":"2024-09-21T10:07:23.501743+0000","last_fullsized":"2024-09-21T10:07:23.501743+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:03.733462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"131'11040","reported_seq":11069,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.273069+0000","last_change":"2024-09-21T10:05:19.214684+0000","last_active":"2024-09-21T10:07:49.273069+0000","last_peered":"2024-09-21T10:07:49.273069+0000","last_clean":"2024-09-21T10:07:49.273069+0000","last_became_active":"2024-09-21T10:05:19.214356+0000","last_became_peered":"2024-09-21T10:05:19.214356+0000","last_unstale":"2024-09-21T10:07:49.273069+0000","last_undegraded":"2024-09-21T10:07:49.273069+0000","last_fullsized":"2024-09-21T10:07:49.273069+0000","mapping_epoch":112,"log_start":"119'3700","ondisk_log_start":"119'3700","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":7340,"log_dups_size":0,"ondisk_log_size":7340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:04.876232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370016256,"num_objects":5646,"num_object_clones":0,"num_object_copies":16938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11040,"num_write_kb":533952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":121,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.636811+0000","last_change":"2024-09-21T10:03:53.845620+0000","last_active":"2024-09-21T10:07:49.636811+0000","last_peered":"2024-09-21T10:07:49.636811+0000","last_clean":"2024-09-21T10:07:49.636811+0000","last_became_active":"2024-09-21T10:03:53.845290+0000","last_became_peered":"2024-09-21T10:03:53.845290+0000","last_unstale":"2024-09-21T10:07:49.636811+0000","last_undegraded":"2024-09-21T10:07:49.636811+0000","last_fullsized":"2024-09-21T10:07:49.636811+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:20.076623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":100,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.574624+0000","last_change":"2024-09-21T10:03:53.846606+0000","last_active":"2024-09-21T10:07:35.574624+0000","last_peered":"2024-09-21T10:07:35.574624+0000","last_clean":"2024-09-21T10:07:35.574624+0000","last_became_active":"2024-09-21T10:03:53.846245+0000","last_became_peered":"2024-09-21T10:03:53.846245+0000","last_unstale":"2024-09-21T10:07:35.574624+0000","last_undegraded":"2024-09-21T10:07:35.574624+0000","last_fullsized":"2024-09-21T10:07:35.574624+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:20.745470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"131'11158","reported_seq":11187,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.096209+0000","last_change":"2024-09-21T10:05:19.210796+0000","last_active":"2024-09-21T10:07:49.096209+0000","last_peered":"2024-09-21T10:07:49.096209+0000","last_clean":"2024-09-21T10:07:49.096209+0000","last_became_active":"2024-09-21T10:05:19.210466+0000","last_became_peered":"2024-09-21T10:05:19.210466+0000","last_unstale":"2024-09-21T10:07:49.096209+0000","last_undegraded":"2024-09-21T10:07:49.096209+0000","last_fullsized":"2024-09-21T10:07:49.096209+0000","mapping_epoch":112,"log_start":"117'2500","ondisk_log_start":"117'2500","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8658,"log_dups_size":0,"ondisk_log_size":8658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:11.178437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367525888,"num_objects":5608,"num_object_clones":0,"num_object_copies":16824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11158,"num_write_kb":536512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":427,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.502826+0000","last_change":"2024-09-21T10:02:12.840225+0000","last_active":"2024-09-21T10:07:23.502826+0000","last_peered":"2024-09-21T10:07:23.502826+0000","last_clean":"2024-09-21T10:07:23.502826+0000","last_became_active":"2024-09-21T10:02:12.839813+0000","last_became_peered":"2024-09-21T10:02:12.839813+0000","last_unstale":"2024-09-21T10:07:23.502826+0000","last_undegraded":"2024-09-21T10:07:23.502826+0000","last_fullsized":"2024-09-21T10:07:23.502826+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:36.630881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":445,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.502442+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:07:23.502442+0000","last_peered":"2024-09-21T10:07:23.502442+0000","last_clean":"2024-09-21T10:07:23.502442+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:07:23.502442+0000","last_undegraded":"2024-09-21T10:07:23.502442+0000","last_fullsized":"2024-09-21T10:07:23.502442+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":68,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.629973+0000","last_change":"2024-09-21T10:03:53.843344+0000","last_active":"2024-09-21T10:07:49.629973+0000","last_peered":"2024-09-21T10:07:49.629973+0000","last_clean":"2024-09-21T10:07:49.629973+0000","last_became_active":"2024-09-21T10:03:53.842304+0000","last_became_peered":"2024-09-21T10:03:53.842304+0000","last_unstale":"2024-09-21T10:07:49.629973+0000","last_undegraded":"2024-09-21T10:07:49.629973+0000","last_fullsized":"2024-09-21T10:07:49.629973+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:31.221735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.506049+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:07:23.506049+0000","last_peered":"2024-09-21T10:07:23.506049+0000","last_clean":"2024-09-21T10:07:23.506049+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:07:23.506049+0000","last_undegraded":"2024-09-21T10:07:23.506049+0000","last_fullsized":"2024-09-21T10:07:23.506049+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"131'11158","reported_seq":11187,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.395793+0000","last_change":"2024-09-21T10:05:19.211638+0000","last_active":"2024-09-21T10:07:49.395793+0000","last_peered":"2024-09-21T10:07:49.395793+0000","last_clean":"2024-09-21T10:07:49.395793+0000","last_became_active":"2024-09-21T10:05:19.211301+0000","last_became_peered":"2024-09-21T10:05:19.211301+0000","last_unstale":"2024-09-21T10:07:49.395793+0000","last_undegraded":"2024-09-21T10:07:49.395793+0000","last_fullsized":"2024-09-21T10:07:49.395793+0000","mapping_epoch":112,"log_start":"117'1100","ondisk_log_start":"117'1100","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10058,"log_dups_size":0,"ondisk_log_size":10058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:53.839733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373161984,"num_objects":5694,"num_object_clones":0,"num_object_copies":17082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11158,"num_write_kb":539264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":591,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:24.146413+0000","last_change":"2024-09-21T10:02:46.663394+0000","last_active":"2024-09-21T10:07:24.146413+0000","last_peered":"2024-09-21T10:07:24.146413+0000","last_clean":"2024-09-21T10:07:24.146413+0000","last_became_active":"2024-09-21T10:02:46.662879+0000","last_became_peered":"2024-09-21T10:02:46.662879+0000","last_unstale":"2024-09-21T10:07:24.146413+0000","last_undegraded":"2024-09-21T10:07:24.146413+0000","last_fullsized":"2024-09-21T10:07:24.146413+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.125795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":110,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.642162+0000","last_change":"2024-09-21T10:03:53.835329+0000","last_active":"2024-09-21T10:07:49.642162+0000","last_peered":"2024-09-21T10:07:49.642162+0000","last_clean":"2024-09-21T10:07:49.642162+0000","last_became_active":"2024-09-21T10:03:53.834934+0000","last_became_peered":"2024-09-21T10:03:53.834934+0000","last_unstale":"2024-09-21T10:07:49.642162+0000","last_undegraded":"2024-09-21T10:07:49.642162+0000","last_fullsized":"2024-09-21T10:07:49.642162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:38.355817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":123,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.643233+0000","last_change":"2024-09-21T10:03:19.770016+0000","last_active":"2024-09-21T10:07:49.643233+0000","last_peered":"2024-09-21T10:07:49.643233+0000","last_clean":"2024-09-21T10:07:49.643233+0000","last_became_active":"2024-09-21T10:03:19.768866+0000","last_became_peered":"2024-09-21T10:03:19.768866+0000","last_unstale":"2024-09-21T10:07:49.643233+0000","last_undegraded":"2024-09-21T10:07:49.643233+0000","last_fullsized":"2024-09-21T10:07:49.643233+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:25.068940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"131'10890","reported_seq":10940,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:48.114411+0000","last_change":"2024-09-21T10:05:19.210830+0000","last_active":"2024-09-21T10:07:48.114411+0000","last_peered":"2024-09-21T10:07:48.114411+0000","last_clean":"2024-09-21T10:07:48.114411+0000","last_became_active":"2024-09-21T10:05:19.210525+0000","last_became_peered":"2024-09-21T10:05:19.210525+0000","last_unstale":"2024-09-21T10:07:48.114411+0000","last_undegraded":"2024-09-21T10:07:48.114411+0000","last_fullsized":"2024-09-21T10:07:48.114411+0000","mapping_epoch":112,"log_start":"117'800","ondisk_log_start":"117'800","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10090,"log_dups_size":0,"ondisk_log_size":10090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:44.771575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370802688,"num_objects":5658,"num_object_clones":0,"num_object_copies":16974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10890,"num_write_kb":529536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":617,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.572056+0000","last_change":"2024-09-21T10:03:53.840322+0000","last_active":"2024-09-21T10:07:35.572056+0000","last_peered":"2024-09-21T10:07:35.572056+0000","last_clean":"2024-09-21T10:07:35.572056+0000","last_became_active":"2024-09-21T10:03:53.839753+0000","last_became_peered":"2024-09-21T10:03:53.839753+0000","last_unstale":"2024-09-21T10:07:35.572056+0000","last_undegraded":"2024-09-21T10:07:35.572056+0000","last_fullsized":"2024-09-21T10:07:35.572056+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:26.364888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":98,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.570590+0000","last_change":"2024-09-21T10:03:19.765402+0000","last_active":"2024-09-21T10:07:35.570590+0000","last_peered":"2024-09-21T10:07:35.570590+0000","last_clean":"2024-09-21T10:07:35.570590+0000","last_became_active":"2024-09-21T10:03:19.765036+0000","last_became_peered":"2024-09-21T10:03:19.765036+0000","last_unstale":"2024-09-21T10:07:35.570590+0000","last_undegraded":"2024-09-21T10:07:35.570590+0000","last_fullsized":"2024-09-21T10:07:35.570590+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:32.230377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":133,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.577590+0000","last_change":"2024-09-21T10:02:46.660814+0000","last_active":"2024-09-21T10:07:35.577590+0000","last_peered":"2024-09-21T10:07:35.577590+0000","last_clean":"2024-09-21T10:07:35.577590+0000","last_became_active":"2024-09-21T10:02:46.660325+0000","last_became_peered":"2024-09-21T10:02:46.660325+0000","last_unstale":"2024-09-21T10:07:35.577590+0000","last_undegraded":"2024-09-21T10:07:35.577590+0000","last_fullsized":"2024-09-21T10:07:35.577590+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:44.427243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"132'11107","reported_seq":11159,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:51.170834+0000","last_change":"2024-09-21T10:05:19.213938+0000","last_active":"2024-09-21T10:07:51.170834+0000","last_peered":"2024-09-21T10:07:51.170834+0000","last_clean":"2024-09-21T10:07:51.170834+0000","last_became_active":"2024-09-21T10:05:19.213559+0000","last_became_peered":"2024-09-21T10:05:19.213559+0000","last_unstale":"2024-09-21T10:07:51.170834+0000","last_undegraded":"2024-09-21T10:07:51.170834+0000","last_fullsized":"2024-09-21T10:07:51.170834+0000","mapping_epoch":112,"log_start":"117'2200","ondisk_log_start":"117'2200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8907,"log_dups_size":0,"ondisk_log_size":8907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:25.094755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363266048,"num_objects":5543,"num_object_clones":0,"num_object_copies":16629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11107,"num_write_kb":532800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":833,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.501131+0000","last_change":"2024-09-21T10:02:46.657107+0000","last_active":"2024-09-21T10:07:23.501131+0000","last_peered":"2024-09-21T10:07:23.501131+0000","last_clean":"2024-09-21T10:07:23.501131+0000","last_became_active":"2024-09-21T10:02:46.656633+0000","last_became_peered":"2024-09-21T10:02:46.656633+0000","last_unstale":"2024-09-21T10:07:23.501131+0000","last_undegraded":"2024-09-21T10:07:23.501131+0000","last_fullsized":"2024-09-21T10:07:23.501131+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:42:18.534832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.500612+0000","last_change":"2024-09-21T10:02:12.840581+0000","last_active":"2024-09-21T10:07:23.500612+0000","last_peered":"2024-09-21T10:07:23.500612+0000","last_clean":"2024-09-21T10:07:23.500612+0000","last_became_active":"2024-09-21T10:02:12.840033+0000","last_became_peered":"2024-09-21T10:02:12.840033+0000","last_unstale":"2024-09-21T10:07:23.500612+0000","last_undegraded":"2024-09-21T10:07:23.500612+0000","last_fullsized":"2024-09-21T10:07:23.500612+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:22.449586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.567349+0000","last_change":"2024-09-21T10:03:53.843551+0000","last_active":"2024-09-21T10:07:35.567349+0000","last_peered":"2024-09-21T10:07:35.567349+0000","last_clean":"2024-09-21T10:07:35.567349+0000","last_became_active":"2024-09-21T10:03:53.843007+0000","last_became_peered":"2024-09-21T10:03:53.843007+0000","last_unstale":"2024-09-21T10:07:35.567349+0000","last_undegraded":"2024-09-21T10:07:35.567349+0000","last_fullsized":"2024-09-21T10:07:35.567349+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:11.757447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"132'11163","reported_seq":11212,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:51.179928+0000","last_change":"2024-09-21T10:05:19.212703+0000","last_active":"2024-09-21T10:07:51.179928+0000","last_peered":"2024-09-21T10:07:51.179928+0000","last_clean":"2024-09-21T10:07:51.179928+0000","last_became_active":"2024-09-21T10:05:19.212420+0000","last_became_peered":"2024-09-21T10:05:19.212420+0000","last_unstale":"2024-09-21T10:07:51.179928+0000","last_undegraded":"2024-09-21T10:07:51.179928+0000","last_fullsized":"2024-09-21T10:07:51.179928+0000","mapping_epoch":112,"log_start":"117'2300","ondisk_log_start":"117'2300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8863,"log_dups_size":0,"ondisk_log_size":8863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:43.062243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360513536,"num_objects":5501,"num_object_clones":0,"num_object_copies":16503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11163,"num_write_kb":533248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":111,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.501027+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:07:23.501027+0000","last_peered":"2024-09-21T10:07:23.501027+0000","last_clean":"2024-09-21T10:07:23.501027+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:07:23.501027+0000","last_undegraded":"2024-09-21T10:07:23.501027+0000","last_fullsized":"2024-09-21T10:07:23.501027+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":573,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.567612+0000","last_change":"2024-09-21T10:02:46.662410+0000","last_active":"2024-09-21T10:07:35.567612+0000","last_peered":"2024-09-21T10:07:35.567612+0000","last_clean":"2024-09-21T10:07:35.567612+0000","last_became_active":"2024-09-21T10:02:46.661506+0000","last_became_peered":"2024-09-21T10:02:46.661506+0000","last_unstale":"2024-09-21T10:07:35.567612+0000","last_undegraded":"2024-09-21T10:07:35.567612+0000","last_fullsized":"2024-09-21T10:07:35.567612+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:40:33.566844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":89,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.567436+0000","last_change":"2024-09-21T10:02:46.665713+0000","last_active":"2024-09-21T10:07:35.567436+0000","last_peered":"2024-09-21T10:07:35.567436+0000","last_clean":"2024-09-21T10:07:35.567436+0000","last_became_active":"2024-09-21T10:02:46.665316+0000","last_became_peered":"2024-09-21T10:02:46.665316+0000","last_unstale":"2024-09-21T10:07:35.567436+0000","last_undegraded":"2024-09-21T10:07:35.567436+0000","last_fullsized":"2024-09-21T10:07:35.567436+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.951202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"131'10933","reported_seq":10962,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.108516+0000","last_change":"2024-09-21T10:05:19.214304+0000","last_active":"2024-09-21T10:07:49.108516+0000","last_peered":"2024-09-21T10:07:49.108516+0000","last_clean":"2024-09-21T10:07:49.108516+0000","last_became_active":"2024-09-21T10:05:19.213946+0000","last_became_peered":"2024-09-21T10:05:19.213946+0000","last_unstale":"2024-09-21T10:07:49.108516+0000","last_undegraded":"2024-09-21T10:07:49.108516+0000","last_fullsized":"2024-09-21T10:07:49.108516+0000","mapping_epoch":112,"log_start":"117'1200","ondisk_log_start":"117'1200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9733,"log_dups_size":0,"ondisk_log_size":9733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:18.695450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362610688,"num_objects":5533,"num_object_clones":0,"num_object_copies":16599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10933,"num_write_kb":526912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":98,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.567789+0000","last_change":"2024-09-21T10:03:19.768187+0000","last_active":"2024-09-21T10:07:35.567789+0000","last_peered":"2024-09-21T10:07:35.567789+0000","last_clean":"2024-09-21T10:07:35.567789+0000","last_became_active":"2024-09-21T10:03:19.767801+0000","last_became_peered":"2024-09-21T10:03:19.767801+0000","last_unstale":"2024-09-21T10:07:35.567789+0000","last_undegraded":"2024-09-21T10:07:35.567789+0000","last_fullsized":"2024-09-21T10:07:35.567789+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:38.042481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":533,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.503718+0000","last_change":"2024-09-21T10:02:12.840296+0000","last_active":"2024-09-21T10:07:23.503718+0000","last_peered":"2024-09-21T10:07:23.503718+0000","last_clean":"2024-09-21T10:07:23.503718+0000","last_became_active":"2024-09-21T10:02:12.839881+0000","last_became_peered":"2024-09-21T10:02:12.839881+0000","last_unstale":"2024-09-21T10:07:23.503718+0000","last_undegraded":"2024-09-21T10:07:23.503718+0000","last_fullsized":"2024-09-21T10:07:23.503718+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:49.166865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":101,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.643370+0000","last_change":"2024-09-21T10:02:12.845158+0000","last_active":"2024-09-21T10:07:49.643370+0000","last_peered":"2024-09-21T10:07:49.643370+0000","last_clean":"2024-09-21T10:07:49.643370+0000","last_became_active":"2024-09-21T10:02:12.843399+0000","last_became_peered":"2024-09-21T10:02:12.843399+0000","last_unstale":"2024-09-21T10:07:49.643370+0000","last_undegraded":"2024-09-21T10:07:49.643370+0000","last_fullsized":"2024-09-21T10:07:49.643370+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:27.552491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"131'10825","reported_seq":10854,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.202823+0000","last_change":"2024-09-21T10:05:19.214110+0000","last_active":"2024-09-21T10:07:49.202823+0000","last_peered":"2024-09-21T10:07:49.202823+0000","last_clean":"2024-09-21T10:07:49.202823+0000","last_became_active":"2024-09-21T10:05:19.213716+0000","last_became_peered":"2024-09-21T10:05:19.213716+0000","last_unstale":"2024-09-21T10:07:49.202823+0000","last_undegraded":"2024-09-21T10:07:49.202823+0000","last_fullsized":"2024-09-21T10:07:49.202823+0000","mapping_epoch":112,"log_start":"118'3500","ondisk_log_start":"118'3500","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:20.723064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372047872,"num_objects":5677,"num_object_clones":0,"num_object_copies":17031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10825,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":118,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.630532+0000","last_change":"2024-09-21T10:03:53.842610+0000","last_active":"2024-09-21T10:07:49.630532+0000","last_peered":"2024-09-21T10:07:49.630532+0000","last_clean":"2024-09-21T10:07:49.630532+0000","last_became_active":"2024-09-21T10:03:53.841888+0000","last_became_peered":"2024-09-21T10:03:53.841888+0000","last_unstale":"2024-09-21T10:07:49.630532+0000","last_undegraded":"2024-09-21T10:07:49.630532+0000","last_fullsized":"2024-09-21T10:07:49.630532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:05.833771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":679,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.567950+0000","last_change":"2024-09-21T10:02:46.665711+0000","last_active":"2024-09-21T10:07:35.567950+0000","last_peered":"2024-09-21T10:07:35.567950+0000","last_clean":"2024-09-21T10:07:35.567950+0000","last_became_active":"2024-09-21T10:02:46.665165+0000","last_became_peered":"2024-09-21T10:02:46.665165+0000","last_unstale":"2024-09-21T10:07:35.567950+0000","last_undegraded":"2024-09-21T10:07:35.567950+0000","last_fullsized":"2024-09-21T10:07:35.567950+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:08.886554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.636685+0000","last_change":"2024-09-21T10:03:53.838941+0000","last_active":"2024-09-21T10:07:49.636685+0000","last_peered":"2024-09-21T10:07:49.636685+0000","last_clean":"2024-09-21T10:07:49.636685+0000","last_became_active":"2024-09-21T10:03:53.838646+0000","last_became_peered":"2024-09-21T10:03:53.838646+0000","last_unstale":"2024-09-21T10:07:49.636685+0000","last_undegraded":"2024-09-21T10:07:49.636685+0000","last_fullsized":"2024-09-21T10:07:49.636685+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:25.167057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"132'11166","reported_seq":11218,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:51.691095+0000","last_change":"2024-09-21T10:05:19.210763+0000","last_active":"2024-09-21T10:07:51.691095+0000","last_peered":"2024-09-21T10:07:51.691095+0000","last_clean":"2024-09-21T10:07:51.691095+0000","last_became_active":"2024-09-21T10:05:19.210462+0000","last_became_peered":"2024-09-21T10:05:19.210462+0000","last_unstale":"2024-09-21T10:07:51.691095+0000","last_undegraded":"2024-09-21T10:07:51.691095+0000","last_fullsized":"2024-09-21T10:07:51.691095+0000","mapping_epoch":112,"log_start":"117'2300","ondisk_log_start":"117'2300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8866,"log_dups_size":0,"ondisk_log_size":8866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:57.912861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362020864,"num_objects":5524,"num_object_clones":0,"num_object_copies":16572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11166,"num_write_kb":534080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.642960+0000","last_change":"2024-09-21T10:02:46.661363+0000","last_active":"2024-09-21T10:07:49.642960+0000","last_peered":"2024-09-21T10:07:49.642960+0000","last_clean":"2024-09-21T10:07:49.642960+0000","last_became_active":"2024-09-21T10:02:46.661005+0000","last_became_peered":"2024-09-21T10:02:46.661005+0000","last_unstale":"2024-09-21T10:07:49.642960+0000","last_undegraded":"2024-09-21T10:07:49.642960+0000","last_fullsized":"2024-09-21T10:07:49.642960+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:42.908471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":371,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.506568+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:07:23.506568+0000","last_peered":"2024-09-21T10:07:23.506568+0000","last_clean":"2024-09-21T10:07:23.506568+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:07:23.506568+0000","last_undegraded":"2024-09-21T10:07:23.506568+0000","last_fullsized":"2024-09-21T10:07:23.506568+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":80,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.504365+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:07:23.504365+0000","last_peered":"2024-09-21T10:07:23.504365+0000","last_clean":"2024-09-21T10:07:23.504365+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:07:23.504365+0000","last_undegraded":"2024-09-21T10:07:23.504365+0000","last_fullsized":"2024-09-21T10:07:23.504365+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":94,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.500436+0000","last_change":"2024-09-21T10:02:46.667831+0000","last_active":"2024-09-21T10:07:23.500436+0000","last_peered":"2024-09-21T10:07:23.500436+0000","last_clean":"2024-09-21T10:07:23.500436+0000","last_became_active":"2024-09-21T10:02:46.667507+0000","last_became_peered":"2024-09-21T10:02:46.667507+0000","last_unstale":"2024-09-21T10:07:23.500436+0000","last_undegraded":"2024-09-21T10:07:23.500436+0000","last_fullsized":"2024-09-21T10:07:23.500436+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:23.990802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":103,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.502048+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:07:23.502048+0000","last_peered":"2024-09-21T10:07:23.502048+0000","last_clean":"2024-09-21T10:07:23.502048+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:07:23.502048+0000","last_undegraded":"2024-09-21T10:07:23.502048+0000","last_fullsized":"2024-09-21T10:07:23.502048+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:29.363995+0000","last_change":"2024-09-21T10:03:19.767147+0000","last_active":"2024-09-21T10:07:29.363995+0000","last_peered":"2024-09-21T10:07:29.363995+0000","last_clean":"2024-09-21T10:07:29.363995+0000","last_became_active":"2024-09-21T10:03:19.766822+0000","last_became_peered":"2024-09-21T10:03:19.766822+0000","last_unstale":"2024-09-21T10:07:29.363995+0000","last_undegraded":"2024-09-21T10:07:29.363995+0000","last_fullsized":"2024-09-21T10:07:29.363995+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:38.875912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.503486+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:07:23.503486+0000","last_peered":"2024-09-21T10:07:23.503486+0000","last_clean":"2024-09-21T10:07:23.503486+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:07:23.503486+0000","last_undegraded":"2024-09-21T10:07:23.503486+0000","last_fullsized":"2024-09-21T10:07:23.503486+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.500755+0000","last_change":"2024-09-21T10:02:46.659383+0000","last_active":"2024-09-21T10:07:23.500755+0000","last_peered":"2024-09-21T10:07:23.500755+0000","last_clean":"2024-09-21T10:07:23.500755+0000","last_became_active":"2024-09-21T10:02:46.659011+0000","last_became_peered":"2024-09-21T10:02:46.659011+0000","last_unstale":"2024-09-21T10:07:23.500755+0000","last_undegraded":"2024-09-21T10:07:23.500755+0000","last_fullsized":"2024-09-21T10:07:23.500755+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:04.650699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":68,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.629485+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:07:49.629485+0000","last_peered":"2024-09-21T10:07:49.629485+0000","last_clean":"2024-09-21T10:07:49.629485+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:07:49.629485+0000","last_undegraded":"2024-09-21T10:07:49.629485+0000","last_fullsized":"2024-09-21T10:07:49.629485+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":110,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.656394+0000","last_change":"2024-09-21T10:03:53.844673+0000","last_active":"2024-09-21T10:07:49.656394+0000","last_peered":"2024-09-21T10:07:49.656394+0000","last_clean":"2024-09-21T10:07:49.656394+0000","last_became_active":"2024-09-21T10:03:53.844313+0000","last_became_peered":"2024-09-21T10:03:53.844313+0000","last_unstale":"2024-09-21T10:07:49.656394+0000","last_undegraded":"2024-09-21T10:07:49.656394+0000","last_fullsized":"2024-09-21T10:07:49.656394+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:42.529301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":133,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.636552+0000","last_change":"2024-09-21T10:03:53.841417+0000","last_active":"2024-09-21T10:07:49.636552+0000","last_peered":"2024-09-21T10:07:49.636552+0000","last_clean":"2024-09-21T10:07:49.636552+0000","last_became_active":"2024-09-21T10:03:53.840993+0000","last_became_peered":"2024-09-21T10:03:53.840993+0000","last_unstale":"2024-09-21T10:07:49.636552+0000","last_undegraded":"2024-09-21T10:07:49.636552+0000","last_fullsized":"2024-09-21T10:07:49.636552+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:05.666490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":104,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.506497+0000","last_change":"2024-09-21T10:03:19.765682+0000","last_active":"2024-09-21T10:07:23.506497+0000","last_peered":"2024-09-21T10:07:23.506497+0000","last_clean":"2024-09-21T10:07:23.506497+0000","last_became_active":"2024-09-21T10:03:19.765344+0000","last_became_peered":"2024-09-21T10:03:19.765344+0000","last_unstale":"2024-09-21T10:07:23.506497+0000","last_undegraded":"2024-09-21T10:07:23.506497+0000","last_fullsized":"2024-09-21T10:07:23.506497+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:43.787280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":136,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.636728+0000","last_change":"2024-09-21T10:03:53.839372+0000","last_active":"2024-09-21T10:07:49.636728+0000","last_peered":"2024-09-21T10:07:49.636728+0000","last_clean":"2024-09-21T10:07:49.636728+0000","last_became_active":"2024-09-21T10:03:53.839053+0000","last_became_peered":"2024-09-21T10:03:53.839053+0000","last_unstale":"2024-09-21T10:07:49.636728+0000","last_undegraded":"2024-09-21T10:07:49.636728+0000","last_fullsized":"2024-09-21T10:07:49.636728+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:29.647982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.643148+0000","last_change":"2024-09-21T10:03:53.844946+0000","last_active":"2024-09-21T10:07:49.643148+0000","last_peered":"2024-09-21T10:07:49.643148+0000","last_clean":"2024-09-21T10:07:49.643148+0000","last_became_active":"2024-09-21T10:03:53.844613+0000","last_became_peered":"2024-09-21T10:03:53.844613+0000","last_unstale":"2024-09-21T10:07:49.643148+0000","last_undegraded":"2024-09-21T10:07:49.643148+0000","last_fullsized":"2024-09-21T10:07:49.643148+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:41.055573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":98,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.566966+0000","last_change":"2024-09-21T10:03:53.846948+0000","last_active":"2024-09-21T10:07:35.566966+0000","last_peered":"2024-09-21T10:07:35.566966+0000","last_clean":"2024-09-21T10:07:35.566966+0000","last_became_active":"2024-09-21T10:03:53.846614+0000","last_became_peered":"2024-09-21T10:03:53.846614+0000","last_unstale":"2024-09-21T10:07:35.566966+0000","last_undegraded":"2024-09-21T10:07:35.566966+0000","last_fullsized":"2024-09-21T10:07:35.566966+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:53.968372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":119,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.629993+0000","last_change":"2024-09-21T10:03:53.845201+0000","last_active":"2024-09-21T10:07:49.629993+0000","last_peered":"2024-09-21T10:07:49.629993+0000","last_clean":"2024-09-21T10:07:49.629993+0000","last_became_active":"2024-09-21T10:03:53.843822+0000","last_became_peered":"2024-09-21T10:03:53.843822+0000","last_unstale":"2024-09-21T10:07:49.629993+0000","last_undegraded":"2024-09-21T10:07:49.629993+0000","last_fullsized":"2024-09-21T10:07:49.629993+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:28.895373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":111,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.561907+0000","last_change":"2024-09-21T10:03:19.768025+0000","last_active":"2024-09-21T10:07:35.561907+0000","last_peered":"2024-09-21T10:07:35.561907+0000","last_clean":"2024-09-21T10:07:35.561907+0000","last_became_active":"2024-09-21T10:03:19.767632+0000","last_became_peered":"2024-09-21T10:03:19.767632+0000","last_unstale":"2024-09-21T10:07:35.561907+0000","last_undegraded":"2024-09-21T10:07:35.561907+0000","last_fullsized":"2024-09-21T10:07:35.561907+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:40.023759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":137,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.572208+0000","last_change":"2024-09-21T10:03:53.840263+0000","last_active":"2024-09-21T10:07:35.572208+0000","last_peered":"2024-09-21T10:07:35.572208+0000","last_clean":"2024-09-21T10:07:35.572208+0000","last_became_active":"2024-09-21T10:03:53.839894+0000","last_became_peered":"2024-09-21T10:03:53.839894+0000","last_unstale":"2024-09-21T10:07:35.572208+0000","last_undegraded":"2024-09-21T10:07:35.572208+0000","last_fullsized":"2024-09-21T10:07:35.572208+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:52.489580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":104,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.643249+0000","last_change":"2024-09-21T10:02:12.846054+0000","last_active":"2024-09-21T10:07:49.643249+0000","last_peered":"2024-09-21T10:07:49.643249+0000","last_clean":"2024-09-21T10:07:49.643249+0000","last_became_active":"2024-09-21T10:02:12.843696+0000","last_became_peered":"2024-09-21T10:02:12.843696+0000","last_unstale":"2024-09-21T10:07:49.643249+0000","last_undegraded":"2024-09-21T10:07:49.643249+0000","last_fullsized":"2024-09-21T10:07:49.643249+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:32.280697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":108,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.500365+0000","last_change":"2024-09-21T10:03:53.840992+0000","last_active":"2024-09-21T10:07:23.500365+0000","last_peered":"2024-09-21T10:07:23.500365+0000","last_clean":"2024-09-21T10:07:23.500365+0000","last_became_active":"2024-09-21T10:03:53.840551+0000","last_became_peered":"2024-09-21T10:03:53.840551+0000","last_unstale":"2024-09-21T10:07:23.500365+0000","last_undegraded":"2024-09-21T10:07:23.500365+0000","last_fullsized":"2024-09-21T10:07:23.500365+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:04.442126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.503396+0000","last_change":"2024-09-21T10:02:46.659513+0000","last_active":"2024-09-21T10:07:23.503396+0000","last_peered":"2024-09-21T10:07:23.503396+0000","last_clean":"2024-09-21T10:07:23.503396+0000","last_became_active":"2024-09-21T10:02:46.659130+0000","last_became_peered":"2024-09-21T10:02:46.659130+0000","last_unstale":"2024-09-21T10:07:23.503396+0000","last_undegraded":"2024-09-21T10:07:23.503396+0000","last_fullsized":"2024-09-21T10:07:23.503396+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:24.792175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":109,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-21T10:07:35.561822+0000","last_change":"2024-09-21T10:03:53.835006+0000","last_active":"2024-09-21T10:07:35.561822+0000","last_peered":"2024-09-21T10:07:35.561822+0000","last_clean":"2024-09-21T10:07:35.561822+0000","last_became_active":"2024-09-21T10:03:53.834608+0000","last_became_peered":"2024-09-21T10:03:53.834608+0000","last_unstale":"2024-09-21T10:07:35.561822+0000","last_undegraded":"2024-09-21T10:07:35.561822+0000","last_fullsized":"2024-09-21T10:07:35.561822+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:56.108606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":81,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.500954+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:07:23.500954+0000","last_peered":"2024-09-21T10:07:23.500954+0000","last_clean":"2024-09-21T10:07:23.500954+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:07:23.500954+0000","last_undegraded":"2024-09-21T10:07:23.500954+0000","last_fullsized":"2024-09-21T10:07:23.500954+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":139,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.629808+0000","last_change":"2024-09-21T10:03:53.844043+0000","last_active":"2024-09-21T10:07:49.629808+0000","last_peered":"2024-09-21T10:07:49.629808+0000","last_clean":"2024-09-21T10:07:49.629808+0000","last_became_active":"2024-09-21T10:03:53.842891+0000","last_became_peered":"2024-09-21T10:03:53.842891+0000","last_unstale":"2024-09-21T10:07:49.629808+0000","last_undegraded":"2024-09-21T10:07:49.629808+0000","last_fullsized":"2024-09-21T10:07:49.629808+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:39.891977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":128,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.630248+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:07:49.630248+0000","last_peered":"2024-09-21T10:07:49.630248+0000","last_clean":"2024-09-21T10:07:49.630248+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:07:49.630248+0000","last_undegraded":"2024-09-21T10:07:49.630248+0000","last_fullsized":"2024-09-21T10:07:49.630248+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":94,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-09-21T10:07:23.500193+0000","last_change":"2024-09-21T10:02:46.659818+0000","last_active":"2024-09-21T10:07:23.500193+0000","last_peered":"2024-09-21T10:07:23.500193+0000","last_clean":"2024-09-21T10:07:23.500193+0000","last_became_active":"2024-09-21T10:02:46.659457+0000","last_became_peered":"2024-09-21T10:02:46.659457+0000","last_unstale":"2024-09-21T10:07:23.500193+0000","last_undegraded":"2024-09-21T10:07:23.500193+0000","last_fullsized":"2024-09-21T10:07:23.500193+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:21.962330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-21T10:07:49.656269+0000","last_change":"2024-09-21T10:03:19.769549+0000","last_active":"2024-09-21T10:07:49.656269+0000","last_peered":"2024-09-21T10:07:49.656269+0000","last_clean":"2024-09-21T10:07:49.656269+0000","last_became_active":"2024-09-21T10:03:19.768854+0000","last_became_peered":"2024-09-21T10:03:19.768854+0000","last_unstale":"2024-09-21T10:07:49.656269+0000","last_undegraded":"2024-09-21T10:07:49.656269+0000","last_fullsized":"2024-09-21T10:07:49.656269+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:29.415039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":5931532312,"num_objects":90509,"num_object_clones":0,"num_object_copies":271527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90509,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":176269,"num_write_kb":8536833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17754566656,"data_stored":17754554440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146769,"ondisk_log_size":146769,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234218,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1922628,"kb_used_data":1833276,"kb_used_omap":1,"kb_used_meta":89342,"kb_avail":91822524,"statfs":{"total":95995035648,"available":94026264576,"internally_reserved":0,"allocated":1877274624,"data_stored":1876578030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":91486667},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495856,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1516220,"kb_used_data":1449012,"kb_used_omap":1,"kb_used_meta":67198,"kb_avail":92228932,"statfs":{"total":95995035648,"available":94442426368,"internally_reserved":0,"allocated":1483788288,"data_stored":1483100066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":68811207},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757493,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3408308,"kb_used_data":3264812,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":90336844,"statfs":{"total":95995035648,"available":92504928256,"internally_reserved":0,"allocated":3343167488,"data_stored":3342482764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":146930118},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019129,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3033404,"kb_used_data":2915956,"kb_used_omap":1,"kb_used_meta":117246,"kb_avail":90711748,"statfs":{"total":95995035648,"available":92888829952,"internally_reserved":0,"allocated":2985938944,"data_stored":2985250235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":120060358},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,4,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435314,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1146296,"kb_used_data":1082668,"kb_used_omap":5,"kb_used_meta":63610,"kb_avail":92598856,"statfs":{"total":95995035648,"available":94821228544,"internally_reserved":0,"allocated":1108652032,"data_stored":1107974943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":65136680},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598840,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2320108,"kb_used_data":2216228,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":91425044,"statfs":{"total":95995035648,"available":93619245056,"internally_reserved":0,"allocated":2269417472,"data_stored":2268740589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":106357197},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762367,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917284,"kb_used_data":1823132,"kb_used_omap":7,"kb_used_meta":94136,"kb_avail":91827868,"statfs":{"total":95995035648,"available":94031736832,"internally_reserved":0,"allocated":1866887168,"data_stored":1866217891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":96395517},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925894,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2958784,"kb_used_data":2846580,"kb_used_omap":7,"kb_used_meta":112184,"kb_avail":90786368,"statfs":{"total":95995035648,"available":92965240832,"internally_reserved":0,"allocated":2914897920,"data_stored":2914213697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":114876606},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2907373568,"data_stored":2907373568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1849491456,"data_stored":1849491456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2253524992,"data_stored":2253520920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1098055680,"data_stored":1098055680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2981625856,"data_stored":2981625856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3328118784,"data_stored":3328114712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1476853760,"data_stored":1476853760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1859522560,"data_stored":1859518488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:07:54.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 1 2024-09-21T10:07:54.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.328 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:54.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:54.429 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:54.473 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:54.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:54.575 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:54.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:54.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:54.723 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:54.761 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:54 smithi045 ceph-mon[95976]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:54.762 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:54.864 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:54.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.923 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:54.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:54.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:54 smithi032 ceph-mon[101299]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:54 smithi032 ceph-mon[105748]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:54.981 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:55.007 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:55.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:55.055 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:07:55.055 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:07:55.082 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:55.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:55.184 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:55.224 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:55.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:55.326 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:55.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.366 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:55.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:55.467 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:55.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.506 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:55.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:55.608 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.646 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:55.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:55.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:55 smithi032 ceph-mon[105748]: pgmap v393: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-09-21T10:07:55.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:55 smithi032 ceph-mon[101299]: pgmap v393: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-09-21T10:07:55.741 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:07:55.747 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:55.838 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:55.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:55.940 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:55.973 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:55.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:55 smithi045 ceph-mon[95976]: pgmap v393: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-09-21T10:07:56.074 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:56.115 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:07:56.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.121 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:56.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:56.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:56.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.253 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:56.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:56.353 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:56.389 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:56.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:56.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:56.527 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:56.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:56.629 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:56.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.671 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:56.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:56.772 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:56.809 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:56.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:56.811 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:07:56.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:56.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.977 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:56.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:57.078 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:57.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:57.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:57.274 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:57.376 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:57.410 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:57.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:57.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:57.538 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:57.639 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:57.685 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:57.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:57.787 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:57.821 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:57.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:57.924 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:57.945 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:57 smithi032 ceph-mon[105748]: pgmap v394: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2024-09-21T10:07:57.946 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:57 smithi032 ceph-mon[101299]: pgmap v394: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2024-09-21T10:07:57.966 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:57.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:57.993 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:57 smithi045 ceph-mon[95976]: pgmap v394: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2024-09-21T10:07:58.068 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.108 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:58.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:58.194 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:07:58.194 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:07:58.209 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:58.250 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:58.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:58.352 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:58.395 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:58.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:58.497 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:58.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:58.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:58.674 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:58.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:58.776 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:58.805 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:58.907 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:58.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:59.049 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:59.050 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:59.090 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:59.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:59.191 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:59.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.229 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:59.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:59.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:59.376 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:07:59.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:59.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:07:59.479 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:59.512 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:59.614 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:59.647 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:59.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:59.672 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:07:59.673 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 1 pgp_num to 1 2024-09-21T10:07:59.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:59.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.780 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:59.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:59.799 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:07:59.823 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[105748]: pgmap v395: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-09-21T10:07:59.823 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3110504399' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:59.823 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:59.823 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2150713630' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:07:59.823 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:07:59.824 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:07:59] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.51.0" 2024-09-21T10:07:59.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[101299]: pgmap v395: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-09-21T10:07:59.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3110504399' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:59.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:59.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2150713630' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:07:59.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:07:59 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:07:59.882 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:59.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.918 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:07:59.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:59.935 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:59 smithi045 ceph-mon[95976]: pgmap v395: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-09-21T10:07:59.935 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3110504399' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:59.935 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:59 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:59.935 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2150713630' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:07:59.936 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:07:59 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:08:00.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:00.054 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:00.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:00.156 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:00.193 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:00.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:00.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:00.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:00.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:00.439 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:00.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.480 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:00.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:00.581 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:00.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.615 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:00.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:00.716 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:00.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:00.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:00.846 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:00 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:00.846 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:00.846 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:08:00.846 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:00 smithi032 ceph-mon[105748]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T10:08:00.846 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:00 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:00.846 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:00.846 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:08:00.847 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:00 smithi032 ceph-mon[101299]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T10:08:00.855 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:00.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:00.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:00.973 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:00 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:00.973 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:00.973 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:08:00.973 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:00 smithi045 ceph-mon[95976]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T10:08:00.991 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:01.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:01.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:01.127 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:01.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:01.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:01.268 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:01.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.317 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:01.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:01.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:01.450 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:01.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:01.552 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:01.586 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:01.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:01.687 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.728 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:01.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:01.828 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:01.859 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:01.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:01.925 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:01 smithi045 ceph-mon[95976]: pgmap v397: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-09-21T10:08:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:01 smithi032 ceph-mon[105748]: pgmap v397: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-09-21T10:08:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:01 smithi032 ceph-mon[101299]: pgmap v397: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-09-21T10:08:01.961 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:01.980 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:01.980 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:02.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.000 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:02.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:02.101 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:02.133 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:02.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:02.235 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:02.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:02.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:02.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:02.406 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:02.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:02.505 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:08:02.505 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:08:02.505 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:08:02.505 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:08:02.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:02.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:02.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:02.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:02.506 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:08:02.506 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:08:02.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:02.507 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:02.507 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:08:02.507 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:02.507 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:02.527 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:08:02.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.535 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:02.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:02.547 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:08:02.547 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:02.636 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:02.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:02.712 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:08:02.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:02.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.817 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:02.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:02.918 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:02.947 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:02.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:02 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2717834975' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:02.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:02 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2717834975' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:03.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:02 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2717834975' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:03.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:03.077 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:03.091 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:03.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:03.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.231 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:03.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:03.332 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:03.373 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:03.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:03.474 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:03.514 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:03.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:03.574 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:03.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:03.652 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:03.754 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:03.884 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.919 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:03.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:03 smithi032 ceph-mon[105748]: pgmap v398: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2024-09-21T10:08:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:03 smithi032 ceph-mon[101299]: pgmap v398: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2024-09-21T10:08:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:03.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:03 smithi045 ceph-mon[95976]: pgmap v398: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2024-09-21T10:08:03.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:04.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:04.053 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:04.155 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:04.190 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:04.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:04.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:04.323 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:04.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:04.425 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:04.463 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:04.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:04.565 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:04.602 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:04.703 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:04.751 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:04.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:04.853 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:04.889 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:04.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:04.990 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:05.034 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:05.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:05.087 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:08:05.087 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:08:05.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.087 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:189290 5.9 5:98f6a969:::benchmark_data_smithi032_140417_object55899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2024-09-21T10:08:05.087 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:05.064239+0000", 2024-09-21T10:08:05.088 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.019770240000000001, 2024-09-21T10:08:05.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.019801302999999999, 2024-09-21T10:08:05.088 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:08:05.088 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:08:05.088 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:08:05.088 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:08:05.088 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:05.088 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 189290 2024-09-21T10:08:05.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.089 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:08:05.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:08:05.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.064239+0000", 2024-09-21T10:08:05.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:08:05.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.064239+0000", 2024-09-21T10:08:05.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:08:05.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.064238+0000", 2024-09-21T10:08:05.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:08:05.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.064244+0000", 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2900000000000002e-06 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.064259+0000", 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5480000000000001e-05 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:08:05.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.064266+0000", 2024-09-21T10:08:05.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0210000000000002e-06 2024-09-21T10:08:05.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:08:05.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.064503+0000", 2024-09-21T10:08:05.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000237066 2024-09-21T10:08:05.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:08:05.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.064797+0000", 2024-09-21T10:08:05.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00029361799999999999 2024-09-21T10:08:05.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:08:05.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.065162+0000", 2024-09-21T10:08:05.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036525099999999998 2024-09-21T10:08:05.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:08:05.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.069927+0000", 2024-09-21T10:08:05.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0047649830000000004 2024-09-21T10:08:05.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:05.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.074707+0000", 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0047796870000000003 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:189295 5.1 e133/112 5:84efd59c:::benchmark_data_smithi032_140417_object55904:head v 133'11658, mlcod=133'11656)", 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:05.074388+0000", 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0096212859999999997, 2024-09-21T10:08:05.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0097412139999999998, 2024-09-21T10:08:05.096 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:08:05.096 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:08:05.096 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:08:05.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:08:05.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.074388+0000", 2024-09-21T10:08:05.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.074388+0000", 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.074387+0000", 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.074397+0000", 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0114e-05 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.074414+0000", 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.666e-05 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.074423+0000", 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7439999999999996e-06 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.074469+0000", 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6638000000000003e-05 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.074489+0000", 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0307000000000001e-05 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:189304 5.4 5:22f7fbbf:::benchmark_data_smithi032_140417_object55913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2024-09-21T10:08:05.099 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:05.075490+0000", 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.008519492, 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087033630000000004, 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 189304 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075490+0000", 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075490+0000", 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075489+0000", 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075495+0000", 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0440000000000001e-06 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075507+0000", 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1897e-05 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075514+0000", 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3030000000000002e-06 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075615+0000", 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000101066 2024-09-21T10:08:05.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075740+0000", 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000124926 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075867+0000", 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012668999999999999 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:05.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080797+0000", 2024-09-21T10:08:05.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049303039999999999 2024-09-21T10:08:05.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:08:05.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080868+0000", 2024-09-21T10:08:05.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0618999999999996e-05 2024-09-21T10:08:05.104 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:189298 5.1 e133/112 5:84ce6d63:::benchmark_data_smithi032_140417_object55907:head v 133'11659, mlcod=133'11656)", 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:05.075531+0000", 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0084781820000000008, 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087468489999999992, 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075531+0000", 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075531+0000", 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075531+0000", 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075535+0000", 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4469999999999999e-06 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075548+0000", 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2832e-05 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:08:05.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.075553+0000", 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8999999999999997e-06 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.077970+0000", 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024173060000000001 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.078013+0000", 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2583000000000002e-05 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44543.0:189299 5.1 e133/112 5:87b08274:::benchmark_data_smithi032_140417_object55908:head v 133'11660, mlcod=133'11656)", 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:05.080498+0000", 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0035115559999999999, 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0038416829999999998, 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080498+0000", 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080498+0000", 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080497+0000", 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080503+0000", 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4040000000000003e-06 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080512+0000", 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8759999999999994e-06 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080518+0000", 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9220000000000002e-06 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.082711+0000", 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0021934290000000002 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.082760+0000", 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8494e-05 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:189307 5.9 5:9c8b8c57:::benchmark_data_smithi032_140417_object55916:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:05.080565+0000", 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0034450169999999999, 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003845968, 2024-09-21T10:08:05.112 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 189307 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080565+0000", 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080565+0000", 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080563+0000", 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080568+0000", 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2379999999999997e-06 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080580+0000", 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1215e-05 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.080587+0000", 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9269999999999999e-06 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081218+0000", 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00063130900000000004 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081339+0000", 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000121511 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081469+0000", 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012991399999999999 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:189308 5.4 5:2a3d34b1:::benchmark_data_smithi032_140417_object55917:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:05.081505+0000", 2024-09-21T10:08:05.116 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0025047329999999999, 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029828110000000001, 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 189308 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081505+0000", 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081505+0000", 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081504+0000", 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081509+0000", 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4820000000000002e-06 2024-09-21T10:08:05.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081521+0000", 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1712000000000001e-05 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081529+0000", 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3769999999999998e-06 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081613+0000", 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4602000000000001e-05 2024-09-21T10:08:05.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081720+0000", 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010692900000000001 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:05.081825+0000", 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000104739 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 7 2024-09-21T10:08:05.120 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:08:05.136 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.178 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:05.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:05.279 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:05.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.319 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:05.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:05.420 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:05.424 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:08:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.454 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:05.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:05.556 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:05.600 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:05.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:05.702 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:05.734 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:05.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:05.790 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:05.836 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:05.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:05.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:05.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:05 smithi032 ceph-mon[105748]: pgmap v399: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-09-21T10:08:05.924 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:05 smithi032 ceph-mon[101299]: pgmap v399: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-09-21T10:08:05.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:06.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:06.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:06.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:05 smithi045 ceph-mon[95976]: pgmap v399: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-09-21T10:08:06.058 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:08:06.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:06.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:06.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:06.218 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:08:06.218 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:08:06.219 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2024-09-21T10:08:06.219 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:08:06.262 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:06.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.317 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:06.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:06.418 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:06.423 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:06.465 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:06.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:06.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.609 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:06.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:06.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:06.727 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:08:06.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:06.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:06.855 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:06.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:06.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:06.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2668449384' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:06.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:06 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2668449384' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:06 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:06.973 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2668449384' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:06.973 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:06 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:06.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.028 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:07.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:07.129 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:07.159 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:07.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:07.305 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:07.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:07.406 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:07.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:07.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:07.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:07.577 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:07.679 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:07.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.718 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:07.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:07.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:07.851 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:07.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:07.952 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:07.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:07 smithi032 ceph-mon[101299]: pgmap v400: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-09-21T10:08:07.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:07 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:07.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:07.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:07 smithi032 ceph-mon[101299]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T10:08:07.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:07 smithi032 ceph-mon[105748]: pgmap v400: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-09-21T10:08:07.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:07 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:07.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:07.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:07 smithi032 ceph-mon[105748]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T10:08:07.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:07 smithi045 ceph-mon[95976]: pgmap v400: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-09-21T10:08:07.977 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:07 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:07.977 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:07.977 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:07 smithi045 ceph-mon[95976]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T10:08:08.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:08.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:08.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:08.136 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:08.238 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:08.277 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:08.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:08.378 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:08.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:08.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:08.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:08.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:08.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:08.650 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:08.674 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:08.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:08.718 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:08:08.718 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:08:08.719 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:08:08.719 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:08:08.719 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:08:08.776 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:08.812 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:08.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:08.914 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:08.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.945 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:08.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:09.045 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:09.078 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:09.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:09.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:09.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:09.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:09.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:09.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.354 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:09.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:09.455 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:09.487 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:09.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:09.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:08:09.588 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:09.599 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:08:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:08:09] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.51.0" 2024-09-21T10:08:09.642 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:09.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:09.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.775 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:09.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:09.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:09 smithi045 ceph-mon[95976]: pgmap v402: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2024-09-21T10:08:09.876 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:09.882 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:09.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:09.950 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:09 smithi032 ceph-mon[105748]: pgmap v402: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2024-09-21T10:08:09.951 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:09 smithi032 ceph-mon[101299]: pgmap v402: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2024-09-21T10:08:10.022 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:10.072 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:10.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:10.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:10.141 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:08:10.141 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:10.171 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:08:10.172 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:10.176 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:10.207 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:08:10.207 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:10.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.211 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:10.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:10.242 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:08:10.242 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:10.271 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:08:10.271 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:10.302 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:08:10.302 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:10.312 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:10.331 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:08:10.331 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:10.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:10.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:10.360 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:08:10.360 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:10.447 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:10.479 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:10.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:10.581 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:10.622 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:10.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:10.724 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:10.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:10.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:10.902 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:10.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:10.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:10 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1017865046' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:10.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:10 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1017865046' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:11.004 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:11.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:10 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1017865046' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:11.046 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:11.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:11.191 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:11.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:11.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:11.325 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:11.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:11.427 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:11.462 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:11.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:11.521 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:08:11.564 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:11.600 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:11.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:11.701 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:11.741 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:11.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:11.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:11.867 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:11.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:11.927 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:11.927 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:11.954 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:11 smithi032 ceph-mon[105748]: pgmap v403: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-21T10:08:11.955 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:11 smithi032 ceph-mon[101299]: pgmap v403: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-21T10:08:11.969 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:12.003 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:12.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:11 smithi045 ceph-mon[95976]: pgmap v403: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-21T10:08:12.105 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:12.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:12.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:12.249 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:12.289 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:12.391 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:12.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.431 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:12.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:12.531 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:12.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.596 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:12.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:12.697 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:12.732 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:12.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:12.833 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:12.868 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:12.969 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:13.006 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:13.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:13.108 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:13.152 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:13.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:13.280 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:08:13.295 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:13.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:13.397 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:13.437 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:13.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:13.538 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:13.579 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:13.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:13.642 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:08:13.681 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:13.719 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:13.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:13.854 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:13.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:13.956 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:13 smithi032 ceph-mon[105748]: pgmap v404: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-21T10:08:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:13 smithi032 ceph-mon[101299]: pgmap v404: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-21T10:08:13.993 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:13.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:13 smithi045 ceph-mon[95976]: pgmap v404: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-21T10:08:14.094 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:14.122 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:14.224 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:14.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:14.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:14.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:14.392 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:14.494 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:14.531 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:14.633 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:14.667 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:14.681 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:08:14.685 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:08:14.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:14.806 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:14.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:14.908 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:14.936 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:14.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:15.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:15.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.076 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:15.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:15.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:15.201 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:15.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:15.303 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:15.343 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:15.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:15.445 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:15.479 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:15.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:15.581 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:15.603 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:15.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:15.704 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:15.746 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:15.848 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:15.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:15.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:15.902 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:15.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:16.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.021 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:16.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:15 smithi045 ceph-mon[95976]: from='client.54556 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:08:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:15 smithi045 ceph-mon[95976]: pgmap v405: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2024-09-21T10:08:16.038 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:15 smithi032 ceph-mon[101299]: from='client.54556 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:08:16.038 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:15 smithi032 ceph-mon[101299]: pgmap v405: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2024-09-21T10:08:16.039 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:15 smithi032 ceph-mon[105748]: from='client.54556 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:08:16.039 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:15 smithi032 ceph-mon[105748]: pgmap v405: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2024-09-21T10:08:16.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:16.155 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:16.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:16.257 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:16.299 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:16.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:16.401 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:16.414 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":404,"stamp":"2024-09-21T10:08:12.200037+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4850522109,"num_objects":73969,"num_object_clones":0,"num_object_copies":221907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73969,"num_whiteouts":0,"num_read":6391,"num_read_kb":13846,"num_write":193960,"num_write_kb":8551874,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":148019,"ondisk_log_size":148019,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15137792,"kb_used_data":14245432,"kb_used_omap":35,"kb_used_meta":892188,"kb_avail":734823424,"statfs":{"total":767960285184,"available":752459186176,"internally_reserved":0,"allocated":14587322368,"data_stored":14581770287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":913601230},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,5,10,8,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-539295744,"num_objects":-8229,"num_object_clones":0,"num_object_copies":-24687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8229,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004641"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":124,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724331+0000","last_change":"2024-09-21T10:03:53.843548+0000","last_active":"2024-09-21T10:08:06.724331+0000","last_peered":"2024-09-21T10:08:06.724331+0000","last_clean":"2024-09-21T10:08:06.724331+0000","last_became_active":"2024-09-21T10:03:53.843182+0000","last_became_peered":"2024-09-21T10:03:53.843182+0000","last_unstale":"2024-09-21T10:08:06.724331+0000","last_undegraded":"2024-09-21T10:08:06.724331+0000","last_fullsized":"2024-09-21T10:08:06.724331+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:09.464209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":100,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.691461+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:07:59.691461+0000","last_peered":"2024-09-21T10:07:59.691461+0000","last_clean":"2024-09-21T10:07:59.691461+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:07:59.691461+0000","last_undegraded":"2024-09-21T10:07:59.691461+0000","last_fullsized":"2024-09-21T10:07:59.691461+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.681748+0000","last_change":"2024-09-21T10:03:53.839987+0000","last_active":"2024-09-21T10:07:59.681748+0000","last_peered":"2024-09-21T10:07:59.681748+0000","last_clean":"2024-09-21T10:07:59.681748+0000","last_became_active":"2024-09-21T10:03:53.839664+0000","last_became_peered":"2024-09-21T10:03:53.839664+0000","last_unstale":"2024-09-21T10:07:59.681748+0000","last_undegraded":"2024-09-21T10:07:59.681748+0000","last_fullsized":"2024-09-21T10:07:59.681748+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:51.726392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":114,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.738466+0000","last_change":"2024-09-21T10:03:53.837496+0000","last_active":"2024-09-21T10:08:06.738466+0000","last_peered":"2024-09-21T10:08:06.738466+0000","last_clean":"2024-09-21T10:08:06.738466+0000","last_became_active":"2024-09-21T10:03:53.837166+0000","last_became_peered":"2024-09-21T10:03:53.837166+0000","last_unstale":"2024-09-21T10:08:06.738466+0000","last_undegraded":"2024-09-21T10:08:06.738466+0000","last_fullsized":"2024-09-21T10:08:06.738466+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:45.795571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.738447+0000","last_change":"2024-09-21T10:03:53.844990+0000","last_active":"2024-09-21T10:08:06.738447+0000","last_peered":"2024-09-21T10:08:06.738447+0000","last_clean":"2024-09-21T10:08:06.738447+0000","last_became_active":"2024-09-21T10:03:53.844664+0000","last_became_peered":"2024-09-21T10:03:53.844664+0000","last_unstale":"2024-09-21T10:08:06.738447+0000","last_undegraded":"2024-09-21T10:08:06.738447+0000","last_fullsized":"2024-09-21T10:08:06.738447+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:01.031522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.695049+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:07:59.695049+0000","last_peered":"2024-09-21T10:07:59.695049+0000","last_clean":"2024-09-21T10:07:59.695049+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:07:59.695049+0000","last_undegraded":"2024-09-21T10:07:59.695049+0000","last_fullsized":"2024-09-21T10:07:59.695049+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":124,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724967+0000","last_change":"2024-09-21T10:03:53.845785+0000","last_active":"2024-09-21T10:08:06.724967+0000","last_peered":"2024-09-21T10:08:06.724967+0000","last_clean":"2024-09-21T10:08:06.724967+0000","last_became_active":"2024-09-21T10:03:53.845303+0000","last_became_peered":"2024-09-21T10:03:53.845303+0000","last_unstale":"2024-09-21T10:08:06.724967+0000","last_undegraded":"2024-09-21T10:08:06.724967+0000","last_fullsized":"2024-09-21T10:08:06.724967+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:04.976007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":106,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.691469+0000","last_change":"2024-09-21T10:02:12.838786+0000","last_active":"2024-09-21T10:07:59.691469+0000","last_peered":"2024-09-21T10:07:59.691469+0000","last_clean":"2024-09-21T10:07:59.691469+0000","last_became_active":"2024-09-21T10:02:12.838242+0000","last_became_peered":"2024-09-21T10:02:12.838242+0000","last_unstale":"2024-09-21T10:07:59.691469+0000","last_undegraded":"2024-09-21T10:07:59.691469+0000","last_fullsized":"2024-09-21T10:07:59.691469+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:31.071136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"134'11998","reported_seq":12028,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:09.196414+0000","last_change":"2024-09-21T10:05:19.211627+0000","last_active":"2024-09-21T10:08:09.196414+0000","last_peered":"2024-09-21T10:08:09.196414+0000","last_clean":"2024-09-21T10:08:09.196414+0000","last_became_active":"2024-09-21T10:05:19.211261+0000","last_became_peered":"2024-09-21T10:05:19.211261+0000","last_unstale":"2024-09-21T10:08:09.196414+0000","last_undegraded":"2024-09-21T10:08:09.196414+0000","last_fullsized":"2024-09-21T10:08:09.196414+0000","mapping_epoch":112,"log_start":"118'3400","ondisk_log_start":"118'3400","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8598,"log_dups_size":0,"ondisk_log_size":8598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:10.360788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307625984,"num_objects":4694,"num_object_clones":0,"num_object_copies":14082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11998,"num_write_kb":534144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.692111+0000","last_change":"2024-09-21T10:02:12.840334+0000","last_active":"2024-09-21T10:07:59.692111+0000","last_peered":"2024-09-21T10:07:59.692111+0000","last_clean":"2024-09-21T10:07:59.692111+0000","last_became_active":"2024-09-21T10:02:12.839928+0000","last_became_peered":"2024-09-21T10:02:12.839928+0000","last_unstale":"2024-09-21T10:07:59.692111+0000","last_undegraded":"2024-09-21T10:07:59.692111+0000","last_fullsized":"2024-09-21T10:07:59.692111+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:38.593644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.680520+0000","last_change":"2024-09-21T09:50:22.931347+0000","last_active":"2024-09-21T10:07:59.680520+0000","last_peered":"2024-09-21T10:07:59.680520+0000","last_clean":"2024-09-21T10:07:59.680520+0000","last_became_active":"2024-09-21T09:50:22.931042+0000","last_became_peered":"2024-09-21T09:50:22.931042+0000","last_unstale":"2024-09-21T10:07:59.680520+0000","last_undegraded":"2024-09-21T10:07:59.680520+0000","last_fullsized":"2024-09-21T10:07:59.680520+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:19.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"134'11935","reported_seq":11991,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:08.187205+0000","last_change":"2024-09-21T10:05:19.213525+0000","last_active":"2024-09-21T10:08:08.187205+0000","last_peered":"2024-09-21T10:08:08.187205+0000","last_clean":"2024-09-21T10:08:08.187205+0000","last_became_active":"2024-09-21T10:05:19.212434+0000","last_became_peered":"2024-09-21T10:05:19.212434+0000","last_unstale":"2024-09-21T10:08:08.187205+0000","last_undegraded":"2024-09-21T10:08:08.187205+0000","last_fullsized":"2024-09-21T10:08:08.187205+0000","mapping_epoch":112,"log_start":"117'1900","ondisk_log_start":"117'1900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10035,"log_dups_size":0,"ondisk_log_size":10035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:35.483451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304939008,"num_objects":4653,"num_object_clones":0,"num_object_copies":13959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11935,"num_write_kb":530816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":100,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.696478+0000","last_change":"2024-09-21T10:02:46.662169+0000","last_active":"2024-09-21T10:07:59.696478+0000","last_peered":"2024-09-21T10:07:59.696478+0000","last_clean":"2024-09-21T10:07:59.696478+0000","last_became_active":"2024-09-21T10:02:46.661721+0000","last_became_peered":"2024-09-21T10:02:46.661721+0000","last_unstale":"2024-09-21T10:07:59.696478+0000","last_undegraded":"2024-09-21T10:07:59.696478+0000","last_fullsized":"2024-09-21T10:07:59.696478+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:10.414152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":113,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.685503+0000","last_change":"2024-09-21T10:03:53.840738+0000","last_active":"2024-09-21T10:07:59.685503+0000","last_peered":"2024-09-21T10:07:59.685503+0000","last_clean":"2024-09-21T10:07:59.685503+0000","last_became_active":"2024-09-21T10:03:53.840363+0000","last_became_peered":"2024-09-21T10:03:53.840363+0000","last_unstale":"2024-09-21T10:07:59.685503+0000","last_undegraded":"2024-09-21T10:07:59.685503+0000","last_fullsized":"2024-09-21T10:07:59.685503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:26.371181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"134'12010","reported_seq":12065,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:08.181180+0000","last_change":"2024-09-21T10:05:19.213181+0000","last_active":"2024-09-21T10:08:08.181180+0000","last_peered":"2024-09-21T10:08:08.181180+0000","last_clean":"2024-09-21T10:08:08.181180+0000","last_became_active":"2024-09-21T10:05:19.212895+0000","last_became_peered":"2024-09-21T10:05:19.212895+0000","last_unstale":"2024-09-21T10:08:08.181180+0000","last_undegraded":"2024-09-21T10:08:08.181180+0000","last_fullsized":"2024-09-21T10:08:08.181180+0000","mapping_epoch":112,"log_start":"117'2000","ondisk_log_start":"117'2000","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10010,"log_dups_size":0,"ondisk_log_size":10010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:40:31.162520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306053120,"num_objects":4670,"num_object_clones":0,"num_object_copies":14010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12010,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.693390+0000","last_change":"2024-09-21T09:50:22.392277+0000","last_active":"2024-09-21T10:07:59.693390+0000","last_peered":"2024-09-21T10:07:59.693390+0000","last_clean":"2024-09-21T10:07:59.693390+0000","last_became_active":"2024-09-21T09:50:22.391952+0000","last_became_peered":"2024-09-21T09:50:22.391952+0000","last_unstale":"2024-09-21T10:07:59.693390+0000","last_undegraded":"2024-09-21T10:07:59.693390+0000","last_fullsized":"2024-09-21T10:07:59.693390+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:57.921001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.692092+0000","last_change":"2024-09-21T10:02:46.663127+0000","last_active":"2024-09-21T10:07:59.692092+0000","last_peered":"2024-09-21T10:07:59.692092+0000","last_clean":"2024-09-21T10:07:59.692092+0000","last_became_active":"2024-09-21T10:02:46.662791+0000","last_became_peered":"2024-09-21T10:02:46.662791+0000","last_unstale":"2024-09-21T10:07:59.692092+0000","last_undegraded":"2024-09-21T10:07:59.692092+0000","last_fullsized":"2024-09-21T10:07:59.692092+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:59.990122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"134'11874","reported_seq":11930,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:08.172787+0000","last_change":"2024-09-21T10:05:19.212279+0000","last_active":"2024-09-21T10:08:08.172787+0000","last_peered":"2024-09-21T10:08:08.172787+0000","last_clean":"2024-09-21T10:08:08.172787+0000","last_became_active":"2024-09-21T10:05:19.211960+0000","last_became_peered":"2024-09-21T10:05:19.211960+0000","last_unstale":"2024-09-21T10:08:08.172787+0000","last_undegraded":"2024-09-21T10:08:08.172787+0000","last_fullsized":"2024-09-21T10:08:08.172787+0000","mapping_epoch":112,"log_start":"117'1800","ondisk_log_start":"117'1800","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:40.251441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315883520,"num_objects":4820,"num_object_clones":0,"num_object_copies":14460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11874,"num_write_kb":534208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.680656+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:07:59.680656+0000","last_peered":"2024-09-21T10:07:59.680656+0000","last_clean":"2024-09-21T10:07:59.680656+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:07:59.680656+0000","last_undegraded":"2024-09-21T10:07:59.680656+0000","last_fullsized":"2024-09-21T10:07:59.680656+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.739144+0000","last_change":"2024-09-21T10:02:46.662651+0000","last_active":"2024-09-21T10:08:06.739144+0000","last_peered":"2024-09-21T10:08:06.739144+0000","last_clean":"2024-09-21T10:08:06.739144+0000","last_became_active":"2024-09-21T10:02:46.662304+0000","last_became_peered":"2024-09-21T10:02:46.662304+0000","last_unstale":"2024-09-21T10:08:06.739144+0000","last_undegraded":"2024-09-21T10:08:06.739144+0000","last_fullsized":"2024-09-21T10:08:06.739144+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:06.746929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"134'12170","reported_seq":12226,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:08.197980+0000","last_change":"2024-09-21T10:05:19.211133+0000","last_active":"2024-09-21T10:08:08.197980+0000","last_peered":"2024-09-21T10:08:08.197980+0000","last_clean":"2024-09-21T10:08:08.197980+0000","last_became_active":"2024-09-21T10:05:19.210822+0000","last_became_peered":"2024-09-21T10:05:19.210822+0000","last_unstale":"2024-09-21T10:08:08.197980+0000","last_undegraded":"2024-09-21T10:08:08.197980+0000","last_fullsized":"2024-09-21T10:08:08.197980+0000","mapping_epoch":112,"log_start":"117'2100","ondisk_log_start":"117'2100","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:02.399086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310050840,"num_objects":4732,"num_object_clones":0,"num_object_copies":14196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4732,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12170,"num_write_kb":540801,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":122,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724331+0000","last_change":"2024-09-21T10:03:53.840511+0000","last_active":"2024-09-21T10:08:06.724331+0000","last_peered":"2024-09-21T10:08:06.724331+0000","last_clean":"2024-09-21T10:08:06.724331+0000","last_became_active":"2024-09-21T10:03:53.840136+0000","last_became_peered":"2024-09-21T10:03:53.840136+0000","last_unstale":"2024-09-21T10:08:06.724331+0000","last_undegraded":"2024-09-21T10:08:06.724331+0000","last_fullsized":"2024-09-21T10:08:06.724331+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:24.765990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.680436+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:07:59.680436+0000","last_peered":"2024-09-21T10:07:59.680436+0000","last_clean":"2024-09-21T10:07:59.680436+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:07:59.680436+0000","last_undegraded":"2024-09-21T10:07:59.680436+0000","last_fullsized":"2024-09-21T10:07:59.680436+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"134'11935","reported_seq":11991,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:09.552272+0000","last_change":"2024-09-21T10:05:19.214310+0000","last_active":"2024-09-21T10:08:09.552272+0000","last_peered":"2024-09-21T10:08:09.552272+0000","last_clean":"2024-09-21T10:08:09.552272+0000","last_became_active":"2024-09-21T10:05:19.213979+0000","last_became_peered":"2024-09-21T10:05:19.213979+0000","last_unstale":"2024-09-21T10:08:09.552272+0000","last_undegraded":"2024-09-21T10:08:09.552272+0000","last_fullsized":"2024-09-21T10:08:09.552272+0000","mapping_epoch":112,"log_start":"119'3800","ondisk_log_start":"119'3800","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8135,"log_dups_size":0,"ondisk_log_size":8135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:05.291457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305856512,"num_objects":4667,"num_object_clones":0,"num_object_copies":14001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11935,"num_write_kb":531264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.692232+0000","last_change":"2024-09-21T10:02:46.663900+0000","last_active":"2024-09-21T10:07:59.692232+0000","last_peered":"2024-09-21T10:07:59.692232+0000","last_clean":"2024-09-21T10:07:59.692232+0000","last_became_active":"2024-09-21T10:02:46.663594+0000","last_became_peered":"2024-09-21T10:02:46.663594+0000","last_unstale":"2024-09-21T10:07:59.692232+0000","last_undegraded":"2024-09-21T10:07:59.692232+0000","last_fullsized":"2024-09-21T10:07:59.692232+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:24.140061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.682847+0000","last_change":"2024-09-21T10:03:19.765623+0000","last_active":"2024-09-21T10:07:59.682847+0000","last_peered":"2024-09-21T10:07:59.682847+0000","last_clean":"2024-09-21T10:07:59.682847+0000","last_became_active":"2024-09-21T10:03:19.765227+0000","last_became_peered":"2024-09-21T10:03:19.765227+0000","last_unstale":"2024-09-21T10:07:59.682847+0000","last_undegraded":"2024-09-21T10:07:59.682847+0000","last_fullsized":"2024-09-21T10:07:59.682847+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:52.227826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"134'12141","reported_seq":12172,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:09.453356+0000","last_change":"2024-09-21T10:05:19.212082+0000","last_active":"2024-09-21T10:08:09.453356+0000","last_peered":"2024-09-21T10:08:09.453356+0000","last_clean":"2024-09-21T10:08:09.453356+0000","last_became_active":"2024-09-21T10:05:19.211673+0000","last_became_peered":"2024-09-21T10:05:19.211673+0000","last_unstale":"2024-09-21T10:08:09.453356+0000","last_undegraded":"2024-09-21T10:08:09.453356+0000","last_fullsized":"2024-09-21T10:08:09.453356+0000","mapping_epoch":112,"log_start":"117'2100","ondisk_log_start":"117'2100","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10041,"log_dups_size":0,"ondisk_log_size":10041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:12.775587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305070080,"num_objects":4655,"num_object_clones":0,"num_object_copies":13965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12141,"num_write_kb":537472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.685647+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:07:59.685647+0000","last_peered":"2024-09-21T10:07:59.685647+0000","last_clean":"2024-09-21T10:07:59.685647+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:07:59.685647+0000","last_undegraded":"2024-09-21T10:07:59.685647+0000","last_fullsized":"2024-09-21T10:07:59.685647+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.681027+0000","last_change":"2024-09-21T10:02:46.664014+0000","last_active":"2024-09-21T10:07:59.681027+0000","last_peered":"2024-09-21T10:07:59.681027+0000","last_clean":"2024-09-21T10:07:59.681027+0000","last_became_active":"2024-09-21T10:02:46.663640+0000","last_became_peered":"2024-09-21T10:02:46.663640+0000","last_unstale":"2024-09-21T10:07:59.681027+0000","last_undegraded":"2024-09-21T10:07:59.681027+0000","last_fullsized":"2024-09-21T10:07:59.681027+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:03.733462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"134'12046","reported_seq":12078,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:09.248877+0000","last_change":"2024-09-21T10:05:19.214684+0000","last_active":"2024-09-21T10:08:09.248877+0000","last_peered":"2024-09-21T10:08:09.248877+0000","last_clean":"2024-09-21T10:08:09.248877+0000","last_became_active":"2024-09-21T10:05:19.214356+0000","last_became_peered":"2024-09-21T10:05:19.214356+0000","last_unstale":"2024-09-21T10:08:09.248877+0000","last_undegraded":"2024-09-21T10:08:09.248877+0000","last_fullsized":"2024-09-21T10:08:09.248877+0000","mapping_epoch":112,"log_start":"121'4700","ondisk_log_start":"121'4700","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":7346,"log_dups_size":0,"ondisk_log_size":7346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:04.876232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304087040,"num_objects":4640,"num_object_clones":0,"num_object_copies":13920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12046,"num_write_kb":533952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":126,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724424+0000","last_change":"2024-09-21T10:03:53.845620+0000","last_active":"2024-09-21T10:08:06.724424+0000","last_peered":"2024-09-21T10:08:06.724424+0000","last_clean":"2024-09-21T10:08:06.724424+0000","last_became_active":"2024-09-21T10:03:53.845290+0000","last_became_peered":"2024-09-21T10:03:53.845290+0000","last_unstale":"2024-09-21T10:08:06.724424+0000","last_undegraded":"2024-09-21T10:08:06.724424+0000","last_fullsized":"2024-09-21T10:08:06.724424+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:20.076623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.735417+0000","last_change":"2024-09-21T10:03:53.846606+0000","last_active":"2024-09-21T10:08:06.735417+0000","last_peered":"2024-09-21T10:08:06.735417+0000","last_clean":"2024-09-21T10:08:06.735417+0000","last_became_active":"2024-09-21T10:03:53.846245+0000","last_became_peered":"2024-09-21T10:03:53.846245+0000","last_unstale":"2024-09-21T10:08:06.735417+0000","last_undegraded":"2024-09-21T10:08:06.735417+0000","last_fullsized":"2024-09-21T10:08:06.735417+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:20.745470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"134'12163","reported_seq":12195,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:09.196432+0000","last_change":"2024-09-21T10:05:19.210796+0000","last_active":"2024-09-21T10:08:09.196432+0000","last_peered":"2024-09-21T10:08:09.196432+0000","last_clean":"2024-09-21T10:08:09.196432+0000","last_became_active":"2024-09-21T10:05:19.210466+0000","last_became_peered":"2024-09-21T10:05:19.210466+0000","last_unstale":"2024-09-21T10:08:09.196432+0000","last_undegraded":"2024-09-21T10:08:09.196432+0000","last_fullsized":"2024-09-21T10:08:09.196432+0000","mapping_epoch":112,"log_start":"118'3500","ondisk_log_start":"118'3500","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8663,"log_dups_size":0,"ondisk_log_size":8663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:11.178437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301662208,"num_objects":4603,"num_object_clones":0,"num_object_copies":13809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12163,"num_write_kb":536512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":430,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.691415+0000","last_change":"2024-09-21T10:02:12.840225+0000","last_active":"2024-09-21T10:07:59.691415+0000","last_peered":"2024-09-21T10:07:59.691415+0000","last_clean":"2024-09-21T10:07:59.691415+0000","last_became_active":"2024-09-21T10:02:12.839813+0000","last_became_peered":"2024-09-21T10:02:12.839813+0000","last_unstale":"2024-09-21T10:07:59.691415+0000","last_undegraded":"2024-09-21T10:07:59.691415+0000","last_fullsized":"2024-09-21T10:07:59.691415+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:36.630881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":449,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.681833+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:07:59.681833+0000","last_peered":"2024-09-21T10:07:59.681833+0000","last_clean":"2024-09-21T10:07:59.681833+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:07:59.681833+0000","last_undegraded":"2024-09-21T10:07:59.681833+0000","last_fullsized":"2024-09-21T10:07:59.681833+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":73,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724305+0000","last_change":"2024-09-21T10:03:53.843344+0000","last_active":"2024-09-21T10:08:06.724305+0000","last_peered":"2024-09-21T10:08:06.724305+0000","last_clean":"2024-09-21T10:08:06.724305+0000","last_became_active":"2024-09-21T10:03:53.842304+0000","last_became_peered":"2024-09-21T10:03:53.842304+0000","last_unstale":"2024-09-21T10:08:06.724305+0000","last_undegraded":"2024-09-21T10:08:06.724305+0000","last_fullsized":"2024-09-21T10:08:06.724305+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:31.221735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.692184+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:07:59.692184+0000","last_peered":"2024-09-21T10:07:59.692184+0000","last_clean":"2024-09-21T10:07:59.692184+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:07:59.692184+0000","last_undegraded":"2024-09-21T10:07:59.692184+0000","last_fullsized":"2024-09-21T10:07:59.692184+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"134'12232","reported_seq":12264,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:09.453137+0000","last_change":"2024-09-21T10:05:19.211638+0000","last_active":"2024-09-21T10:08:09.453137+0000","last_peered":"2024-09-21T10:08:09.453137+0000","last_clean":"2024-09-21T10:08:09.453137+0000","last_became_active":"2024-09-21T10:05:19.211301+0000","last_became_peered":"2024-09-21T10:05:19.211301+0000","last_unstale":"2024-09-21T10:08:09.453137+0000","last_undegraded":"2024-09-21T10:08:09.453137+0000","last_fullsized":"2024-09-21T10:08:09.453137+0000","mapping_epoch":112,"log_start":"117'2200","ondisk_log_start":"117'2200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10032,"log_dups_size":0,"ondisk_log_size":10032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:53.839733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302776320,"num_objects":4620,"num_object_clones":0,"num_object_copies":13860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12232,"num_write_kb":539264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":594,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.691306+0000","last_change":"2024-09-21T10:02:46.663394+0000","last_active":"2024-09-21T10:07:59.691306+0000","last_peered":"2024-09-21T10:07:59.691306+0000","last_clean":"2024-09-21T10:07:59.691306+0000","last_became_active":"2024-09-21T10:02:46.662879+0000","last_became_peered":"2024-09-21T10:02:46.662879+0000","last_unstale":"2024-09-21T10:07:59.691306+0000","last_undegraded":"2024-09-21T10:07:59.691306+0000","last_fullsized":"2024-09-21T10:07:59.691306+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.125795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":114,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.737962+0000","last_change":"2024-09-21T10:03:53.835329+0000","last_active":"2024-09-21T10:08:06.737962+0000","last_peered":"2024-09-21T10:08:06.737962+0000","last_clean":"2024-09-21T10:08:06.737962+0000","last_became_active":"2024-09-21T10:03:53.834934+0000","last_became_peered":"2024-09-21T10:03:53.834934+0000","last_unstale":"2024-09-21T10:08:06.737962+0000","last_undegraded":"2024-09-21T10:08:06.737962+0000","last_fullsized":"2024-09-21T10:08:06.737962+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:38.355817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.738628+0000","last_change":"2024-09-21T10:03:19.770016+0000","last_active":"2024-09-21T10:08:06.738628+0000","last_peered":"2024-09-21T10:08:06.738628+0000","last_clean":"2024-09-21T10:08:06.738628+0000","last_became_active":"2024-09-21T10:03:19.768866+0000","last_became_peered":"2024-09-21T10:03:19.768866+0000","last_unstale":"2024-09-21T10:08:06.738628+0000","last_undegraded":"2024-09-21T10:08:06.738628+0000","last_fullsized":"2024-09-21T10:08:06.738628+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:25.068940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"134'11946","reported_seq":12001,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:08.201465+0000","last_change":"2024-09-21T10:05:19.210830+0000","last_active":"2024-09-21T10:08:08.201465+0000","last_peered":"2024-09-21T10:08:08.201465+0000","last_clean":"2024-09-21T10:08:08.201465+0000","last_became_active":"2024-09-21T10:05:19.210525+0000","last_became_peered":"2024-09-21T10:05:19.210525+0000","last_unstale":"2024-09-21T10:08:08.201465+0000","last_undegraded":"2024-09-21T10:08:08.201465+0000","last_fullsized":"2024-09-21T10:08:08.201465+0000","mapping_epoch":112,"log_start":"117'1900","ondisk_log_start":"117'1900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":10046,"log_dups_size":0,"ondisk_log_size":10046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:44.771575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301596672,"num_objects":4602,"num_object_clones":0,"num_object_copies":13806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11946,"num_write_kb":529536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":623,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.732769+0000","last_change":"2024-09-21T10:03:53.840322+0000","last_active":"2024-09-21T10:08:06.732769+0000","last_peered":"2024-09-21T10:08:06.732769+0000","last_clean":"2024-09-21T10:08:06.732769+0000","last_became_active":"2024-09-21T10:03:53.839753+0000","last_became_peered":"2024-09-21T10:03:53.839753+0000","last_unstale":"2024-09-21T10:08:06.732769+0000","last_undegraded":"2024-09-21T10:08:06.732769+0000","last_fullsized":"2024-09-21T10:08:06.732769+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:26.364888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":104,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.735173+0000","last_change":"2024-09-21T10:03:19.765402+0000","last_active":"2024-09-21T10:08:06.735173+0000","last_peered":"2024-09-21T10:08:06.735173+0000","last_clean":"2024-09-21T10:08:06.735173+0000","last_became_active":"2024-09-21T10:03:19.765036+0000","last_became_peered":"2024-09-21T10:03:19.765036+0000","last_unstale":"2024-09-21T10:08:06.735173+0000","last_undegraded":"2024-09-21T10:08:06.735173+0000","last_fullsized":"2024-09-21T10:08:06.735173+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:32.230377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":139,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.734908+0000","last_change":"2024-09-21T10:02:46.660814+0000","last_active":"2024-09-21T10:08:06.734908+0000","last_peered":"2024-09-21T10:08:06.734908+0000","last_clean":"2024-09-21T10:08:06.734908+0000","last_became_active":"2024-09-21T10:02:46.660325+0000","last_became_peered":"2024-09-21T10:02:46.660325+0000","last_unstale":"2024-09-21T10:08:06.734908+0000","last_undegraded":"2024-09-21T10:08:06.734908+0000","last_fullsized":"2024-09-21T10:08:06.734908+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:44.427243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"134'12143","reported_seq":12199,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:11.170657+0000","last_change":"2024-09-21T10:05:19.213938+0000","last_active":"2024-09-21T10:08:11.170657+0000","last_peered":"2024-09-21T10:08:11.170657+0000","last_clean":"2024-09-21T10:08:11.170657+0000","last_became_active":"2024-09-21T10:05:19.213559+0000","last_became_peered":"2024-09-21T10:05:19.213559+0000","last_unstale":"2024-09-21T10:08:11.170657+0000","last_undegraded":"2024-09-21T10:08:11.170657+0000","last_fullsized":"2024-09-21T10:08:11.170657+0000","mapping_epoch":112,"log_start":"118'3300","ondisk_log_start":"118'3300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8843,"log_dups_size":0,"ondisk_log_size":8843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:25.094755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295370752,"num_objects":4507,"num_object_clones":0,"num_object_copies":13521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12143,"num_write_kb":532800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":837,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.684760+0000","last_change":"2024-09-21T10:02:46.657107+0000","last_active":"2024-09-21T10:07:59.684760+0000","last_peered":"2024-09-21T10:07:59.684760+0000","last_clean":"2024-09-21T10:07:59.684760+0000","last_became_active":"2024-09-21T10:02:46.656633+0000","last_became_peered":"2024-09-21T10:02:46.656633+0000","last_unstale":"2024-09-21T10:07:59.684760+0000","last_undegraded":"2024-09-21T10:07:59.684760+0000","last_fullsized":"2024-09-21T10:07:59.684760+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:42:18.534832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.680658+0000","last_change":"2024-09-21T10:02:12.840581+0000","last_active":"2024-09-21T10:07:59.680658+0000","last_peered":"2024-09-21T10:07:59.680658+0000","last_clean":"2024-09-21T10:07:59.680658+0000","last_became_active":"2024-09-21T10:02:12.840033+0000","last_became_peered":"2024-09-21T10:02:12.840033+0000","last_unstale":"2024-09-21T10:07:59.680658+0000","last_undegraded":"2024-09-21T10:07:59.680658+0000","last_fullsized":"2024-09-21T10:07:59.680658+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:22.449586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":117,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.735220+0000","last_change":"2024-09-21T10:03:53.843551+0000","last_active":"2024-09-21T10:08:06.735220+0000","last_peered":"2024-09-21T10:08:06.735220+0000","last_clean":"2024-09-21T10:08:06.735220+0000","last_became_active":"2024-09-21T10:03:53.843007+0000","last_became_peered":"2024-09-21T10:03:53.843007+0000","last_unstale":"2024-09-21T10:08:06.735220+0000","last_undegraded":"2024-09-21T10:08:06.735220+0000","last_fullsized":"2024-09-21T10:08:06.735220+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:11.757447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"134'12202","reported_seq":12258,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:11.184255+0000","last_change":"2024-09-21T10:05:19.212703+0000","last_active":"2024-09-21T10:08:11.184255+0000","last_peered":"2024-09-21T10:08:11.184255+0000","last_clean":"2024-09-21T10:08:11.184255+0000","last_became_active":"2024-09-21T10:05:19.212420+0000","last_became_peered":"2024-09-21T10:05:19.212420+0000","last_unstale":"2024-09-21T10:08:11.184255+0000","last_undegraded":"2024-09-21T10:08:11.184255+0000","last_fullsized":"2024-09-21T10:08:11.184255+0000","mapping_epoch":112,"log_start":"118'3300","ondisk_log_start":"118'3300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8902,"log_dups_size":0,"ondisk_log_size":8902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:43.062243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292421632,"num_objects":4462,"num_object_clones":0,"num_object_copies":13386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12202,"num_write_kb":533248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":114,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.681844+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:07:59.681844+0000","last_peered":"2024-09-21T10:07:59.681844+0000","last_clean":"2024-09-21T10:07:59.681844+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:07:59.681844+0000","last_undegraded":"2024-09-21T10:07:59.681844+0000","last_fullsized":"2024-09-21T10:07:59.681844+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":579,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.735318+0000","last_change":"2024-09-21T10:02:46.662410+0000","last_active":"2024-09-21T10:08:06.735318+0000","last_peered":"2024-09-21T10:08:06.735318+0000","last_clean":"2024-09-21T10:08:06.735318+0000","last_became_active":"2024-09-21T10:02:46.661506+0000","last_became_peered":"2024-09-21T10:02:46.661506+0000","last_unstale":"2024-09-21T10:08:06.735318+0000","last_undegraded":"2024-09-21T10:08:06.735318+0000","last_fullsized":"2024-09-21T10:08:06.735318+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:40:33.566844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":95,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.734985+0000","last_change":"2024-09-21T10:02:46.665713+0000","last_active":"2024-09-21T10:08:06.734985+0000","last_peered":"2024-09-21T10:08:06.734985+0000","last_clean":"2024-09-21T10:08:06.734985+0000","last_became_active":"2024-09-21T10:02:46.665316+0000","last_became_peered":"2024-09-21T10:02:46.665316+0000","last_unstale":"2024-09-21T10:08:06.734985+0000","last_undegraded":"2024-09-21T10:08:06.734985+0000","last_fullsized":"2024-09-21T10:08:06.734985+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.951202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"134'11968","reported_seq":12000,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:09.119780+0000","last_change":"2024-09-21T10:05:19.214304+0000","last_active":"2024-09-21T10:08:09.119780+0000","last_peered":"2024-09-21T10:08:09.119780+0000","last_clean":"2024-09-21T10:08:09.119780+0000","last_became_active":"2024-09-21T10:05:19.213946+0000","last_became_peered":"2024-09-21T10:05:19.213946+0000","last_unstale":"2024-09-21T10:08:09.119780+0000","last_undegraded":"2024-09-21T10:08:09.119780+0000","last_fullsized":"2024-09-21T10:08:09.119780+0000","mapping_epoch":112,"log_start":"117'2200","ondisk_log_start":"117'2200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":9768,"log_dups_size":0,"ondisk_log_size":9768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:18.695450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294780928,"num_objects":4498,"num_object_clones":0,"num_object_copies":13494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11968,"num_write_kb":526912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":104,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.735521+0000","last_change":"2024-09-21T10:03:19.768187+0000","last_active":"2024-09-21T10:08:06.735521+0000","last_peered":"2024-09-21T10:08:06.735521+0000","last_clean":"2024-09-21T10:08:06.735521+0000","last_became_active":"2024-09-21T10:03:19.767801+0000","last_became_peered":"2024-09-21T10:03:19.767801+0000","last_unstale":"2024-09-21T10:08:06.735521+0000","last_undegraded":"2024-09-21T10:08:06.735521+0000","last_fullsized":"2024-09-21T10:08:06.735521+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:38.042481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":537,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.729210+0000","last_change":"2024-09-21T10:02:12.840296+0000","last_active":"2024-09-21T10:08:06.729210+0000","last_peered":"2024-09-21T10:08:06.729210+0000","last_clean":"2024-09-21T10:08:06.729210+0000","last_became_active":"2024-09-21T10:02:12.839881+0000","last_became_peered":"2024-09-21T10:02:12.839881+0000","last_unstale":"2024-09-21T10:08:06.729210+0000","last_undegraded":"2024-09-21T10:08:06.729210+0000","last_fullsized":"2024-09-21T10:08:06.729210+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:49.166865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":105,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.739064+0000","last_change":"2024-09-21T10:02:12.845158+0000","last_active":"2024-09-21T10:08:06.739064+0000","last_peered":"2024-09-21T10:08:06.739064+0000","last_clean":"2024-09-21T10:08:06.739064+0000","last_became_active":"2024-09-21T10:02:12.843399+0000","last_became_peered":"2024-09-21T10:02:12.843399+0000","last_unstale":"2024-09-21T10:08:06.739064+0000","last_undegraded":"2024-09-21T10:08:06.739064+0000","last_fullsized":"2024-09-21T10:08:06.739064+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:27.552491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"134'11879","reported_seq":11911,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:09.236866+0000","last_change":"2024-09-21T10:05:19.214110+0000","last_active":"2024-09-21T10:08:09.236866+0000","last_peered":"2024-09-21T10:08:09.236866+0000","last_clean":"2024-09-21T10:08:09.236866+0000","last_became_active":"2024-09-21T10:05:19.213716+0000","last_became_peered":"2024-09-21T10:05:19.213716+0000","last_unstale":"2024-09-21T10:08:09.236866+0000","last_undegraded":"2024-09-21T10:08:09.236866+0000","last_fullsized":"2024-09-21T10:08:09.236866+0000","mapping_epoch":112,"log_start":"121'4500","ondisk_log_start":"121'4500","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":7379,"log_dups_size":0,"ondisk_log_size":7379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:20.723064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302972928,"num_objects":4623,"num_object_clones":0,"num_object_copies":13869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11879,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":123,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.725134+0000","last_change":"2024-09-21T10:03:53.842610+0000","last_active":"2024-09-21T10:08:06.725134+0000","last_peered":"2024-09-21T10:08:06.725134+0000","last_clean":"2024-09-21T10:08:06.725134+0000","last_became_active":"2024-09-21T10:03:53.841888+0000","last_became_peered":"2024-09-21T10:03:53.841888+0000","last_unstale":"2024-09-21T10:08:06.725134+0000","last_undegraded":"2024-09-21T10:08:06.725134+0000","last_fullsized":"2024-09-21T10:08:06.725134+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:05.833771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":685,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.735091+0000","last_change":"2024-09-21T10:02:46.665711+0000","last_active":"2024-09-21T10:08:06.735091+0000","last_peered":"2024-09-21T10:08:06.735091+0000","last_clean":"2024-09-21T10:08:06.735091+0000","last_became_active":"2024-09-21T10:02:46.665165+0000","last_became_peered":"2024-09-21T10:02:46.665165+0000","last_unstale":"2024-09-21T10:08:06.735091+0000","last_undegraded":"2024-09-21T10:08:06.735091+0000","last_fullsized":"2024-09-21T10:08:06.735091+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:08.886554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":120,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724180+0000","last_change":"2024-09-21T10:03:53.838941+0000","last_active":"2024-09-21T10:08:06.724180+0000","last_peered":"2024-09-21T10:08:06.724180+0000","last_clean":"2024-09-21T10:08:06.724180+0000","last_became_active":"2024-09-21T10:03:53.838646+0000","last_became_peered":"2024-09-21T10:03:53.838646+0000","last_unstale":"2024-09-21T10:08:06.724180+0000","last_undegraded":"2024-09-21T10:08:06.724180+0000","last_fullsized":"2024-09-21T10:08:06.724180+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:25.167057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"134'12195","reported_seq":12251,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:11.660266+0000","last_change":"2024-09-21T10:05:19.210763+0000","last_active":"2024-09-21T10:08:11.660266+0000","last_peered":"2024-09-21T10:08:11.660266+0000","last_clean":"2024-09-21T10:08:11.660266+0000","last_became_active":"2024-09-21T10:05:19.210462+0000","last_became_peered":"2024-09-21T10:05:19.210462+0000","last_unstale":"2024-09-21T10:08:11.660266+0000","last_undegraded":"2024-09-21T10:08:11.660266+0000","last_fullsized":"2024-09-21T10:08:11.660266+0000","mapping_epoch":112,"log_start":"118'3300","ondisk_log_start":"118'3300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:05:18.192366+0000","last_clean_scrub_stamp":"2024-09-21T10:05:18.192366+0000","objects_scrubbed":0,"log_size":8895,"log_dups_size":0,"ondisk_log_size":8895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:57.912861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294584320,"num_objects":4495,"num_object_clones":0,"num_object_copies":13485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12195,"num_write_kb":534080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":119,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.738286+0000","last_change":"2024-09-21T10:02:46.661363+0000","last_active":"2024-09-21T10:08:06.738286+0000","last_peered":"2024-09-21T10:08:06.738286+0000","last_clean":"2024-09-21T10:08:06.738286+0000","last_became_active":"2024-09-21T10:02:46.661005+0000","last_became_peered":"2024-09-21T10:02:46.661005+0000","last_unstale":"2024-09-21T10:08:06.738286+0000","last_undegraded":"2024-09-21T10:08:06.738286+0000","last_fullsized":"2024-09-21T10:08:06.738286+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:42.908471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":374,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.694346+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:07:59.694346+0000","last_peered":"2024-09-21T10:07:59.694346+0000","last_clean":"2024-09-21T10:07:59.694346+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:07:59.694346+0000","last_undegraded":"2024-09-21T10:07:59.694346+0000","last_fullsized":"2024-09-21T10:07:59.694346+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.685553+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:07:59.685553+0000","last_peered":"2024-09-21T10:07:59.685553+0000","last_clean":"2024-09-21T10:07:59.685553+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:07:59.685553+0000","last_undegraded":"2024-09-21T10:07:59.685553+0000","last_fullsized":"2024-09-21T10:07:59.685553+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.680453+0000","last_change":"2024-09-21T10:02:46.667831+0000","last_active":"2024-09-21T10:07:59.680453+0000","last_peered":"2024-09-21T10:07:59.680453+0000","last_clean":"2024-09-21T10:07:59.680453+0000","last_became_active":"2024-09-21T10:02:46.667507+0000","last_became_peered":"2024-09-21T10:02:46.667507+0000","last_unstale":"2024-09-21T10:07:59.680453+0000","last_undegraded":"2024-09-21T10:07:59.680453+0000","last_fullsized":"2024-09-21T10:07:59.680453+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:23.990802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":106,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.680602+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:07:59.680602+0000","last_peered":"2024-09-21T10:07:59.680602+0000","last_clean":"2024-09-21T10:07:59.680602+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:07:59.680602+0000","last_undegraded":"2024-09-21T10:07:59.680602+0000","last_fullsized":"2024-09-21T10:07:59.680602+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":94,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.684368+0000","last_change":"2024-09-21T10:03:19.767147+0000","last_active":"2024-09-21T10:07:59.684368+0000","last_peered":"2024-09-21T10:07:59.684368+0000","last_clean":"2024-09-21T10:07:59.684368+0000","last_became_active":"2024-09-21T10:03:19.766822+0000","last_became_peered":"2024-09-21T10:03:19.766822+0000","last_unstale":"2024-09-21T10:07:59.684368+0000","last_undegraded":"2024-09-21T10:07:59.684368+0000","last_fullsized":"2024-09-21T10:07:59.684368+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:38.875912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":101,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.691723+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:07:59.691723+0000","last_peered":"2024-09-21T10:07:59.691723+0000","last_clean":"2024-09-21T10:07:59.691723+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:07:59.691723+0000","last_undegraded":"2024-09-21T10:07:59.691723+0000","last_fullsized":"2024-09-21T10:07:59.691723+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.682684+0000","last_change":"2024-09-21T10:02:46.659383+0000","last_active":"2024-09-21T10:07:59.682684+0000","last_peered":"2024-09-21T10:07:59.682684+0000","last_clean":"2024-09-21T10:07:59.682684+0000","last_became_active":"2024-09-21T10:02:46.659011+0000","last_became_peered":"2024-09-21T10:02:46.659011+0000","last_unstale":"2024-09-21T10:07:59.682684+0000","last_undegraded":"2024-09-21T10:07:59.682684+0000","last_fullsized":"2024-09-21T10:07:59.682684+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:04.650699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":72,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724240+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:08:06.724240+0000","last_peered":"2024-09-21T10:08:06.724240+0000","last_clean":"2024-09-21T10:08:06.724240+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:08:06.724240+0000","last_undegraded":"2024-09-21T10:08:06.724240+0000","last_fullsized":"2024-09-21T10:08:06.724240+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":114,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.738541+0000","last_change":"2024-09-21T10:03:53.844673+0000","last_active":"2024-09-21T10:08:06.738541+0000","last_peered":"2024-09-21T10:08:06.738541+0000","last_clean":"2024-09-21T10:08:06.738541+0000","last_became_active":"2024-09-21T10:03:53.844313+0000","last_became_peered":"2024-09-21T10:03:53.844313+0000","last_unstale":"2024-09-21T10:08:06.738541+0000","last_undegraded":"2024-09-21T10:08:06.738541+0000","last_fullsized":"2024-09-21T10:08:06.738541+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:42.529301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":137,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724133+0000","last_change":"2024-09-21T10:03:53.841417+0000","last_active":"2024-09-21T10:08:06.724133+0000","last_peered":"2024-09-21T10:08:06.724133+0000","last_clean":"2024-09-21T10:08:06.724133+0000","last_became_active":"2024-09-21T10:03:53.840993+0000","last_became_peered":"2024-09-21T10:03:53.840993+0000","last_unstale":"2024-09-21T10:08:06.724133+0000","last_undegraded":"2024-09-21T10:08:06.724133+0000","last_fullsized":"2024-09-21T10:08:06.724133+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:05.666490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.694254+0000","last_change":"2024-09-21T10:03:19.765682+0000","last_active":"2024-09-21T10:07:59.694254+0000","last_peered":"2024-09-21T10:07:59.694254+0000","last_clean":"2024-09-21T10:07:59.694254+0000","last_became_active":"2024-09-21T10:03:19.765344+0000","last_became_peered":"2024-09-21T10:03:19.765344+0000","last_unstale":"2024-09-21T10:07:59.694254+0000","last_undegraded":"2024-09-21T10:07:59.694254+0000","last_fullsized":"2024-09-21T10:07:59.694254+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:43.787280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":140,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724277+0000","last_change":"2024-09-21T10:03:53.839372+0000","last_active":"2024-09-21T10:08:06.724277+0000","last_peered":"2024-09-21T10:08:06.724277+0000","last_clean":"2024-09-21T10:08:06.724277+0000","last_became_active":"2024-09-21T10:03:53.839053+0000","last_became_peered":"2024-09-21T10:03:53.839053+0000","last_unstale":"2024-09-21T10:08:06.724277+0000","last_undegraded":"2024-09-21T10:08:06.724277+0000","last_fullsized":"2024-09-21T10:08:06.724277+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:29.647982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":117,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.738576+0000","last_change":"2024-09-21T10:03:53.844946+0000","last_active":"2024-09-21T10:08:06.738576+0000","last_peered":"2024-09-21T10:08:06.738576+0000","last_clean":"2024-09-21T10:08:06.738576+0000","last_became_active":"2024-09-21T10:03:53.844613+0000","last_became_peered":"2024-09-21T10:03:53.844613+0000","last_unstale":"2024-09-21T10:08:06.738576+0000","last_undegraded":"2024-09-21T10:08:06.738576+0000","last_fullsized":"2024-09-21T10:08:06.738576+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:41.055573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":104,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.734745+0000","last_change":"2024-09-21T10:03:53.846948+0000","last_active":"2024-09-21T10:08:06.734745+0000","last_peered":"2024-09-21T10:08:06.734745+0000","last_clean":"2024-09-21T10:08:06.734745+0000","last_became_active":"2024-09-21T10:03:53.846614+0000","last_became_peered":"2024-09-21T10:03:53.846614+0000","last_unstale":"2024-09-21T10:08:06.734745+0000","last_undegraded":"2024-09-21T10:08:06.734745+0000","last_fullsized":"2024-09-21T10:08:06.734745+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:53.968372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":123,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724769+0000","last_change":"2024-09-21T10:03:53.845201+0000","last_active":"2024-09-21T10:08:06.724769+0000","last_peered":"2024-09-21T10:08:06.724769+0000","last_clean":"2024-09-21T10:08:06.724769+0000","last_became_active":"2024-09-21T10:03:53.843822+0000","last_became_peered":"2024-09-21T10:03:53.843822+0000","last_unstale":"2024-09-21T10:08:06.724769+0000","last_undegraded":"2024-09-21T10:08:06.724769+0000","last_fullsized":"2024-09-21T10:08:06.724769+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:28.895373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":114,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.681636+0000","last_change":"2024-09-21T10:03:19.768025+0000","last_active":"2024-09-21T10:07:59.681636+0000","last_peered":"2024-09-21T10:07:59.681636+0000","last_clean":"2024-09-21T10:07:59.681636+0000","last_became_active":"2024-09-21T10:03:19.767632+0000","last_became_peered":"2024-09-21T10:03:19.767632+0000","last_unstale":"2024-09-21T10:07:59.681636+0000","last_undegraded":"2024-09-21T10:07:59.681636+0000","last_fullsized":"2024-09-21T10:07:59.681636+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:40.023759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":143,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.733028+0000","last_change":"2024-09-21T10:03:53.840263+0000","last_active":"2024-09-21T10:08:06.733028+0000","last_peered":"2024-09-21T10:08:06.733028+0000","last_clean":"2024-09-21T10:08:06.733028+0000","last_became_active":"2024-09-21T10:03:53.839894+0000","last_became_peered":"2024-09-21T10:03:53.839894+0000","last_unstale":"2024-09-21T10:08:06.733028+0000","last_undegraded":"2024-09-21T10:08:06.733028+0000","last_fullsized":"2024-09-21T10:08:06.733028+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:52.489580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":108,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.738979+0000","last_change":"2024-09-21T10:02:12.846054+0000","last_active":"2024-09-21T10:08:06.738979+0000","last_peered":"2024-09-21T10:08:06.738979+0000","last_clean":"2024-09-21T10:08:06.738979+0000","last_became_active":"2024-09-21T10:02:12.843696+0000","last_became_peered":"2024-09-21T10:02:12.843696+0000","last_unstale":"2024-09-21T10:08:06.738979+0000","last_undegraded":"2024-09-21T10:08:06.738979+0000","last_fullsized":"2024-09-21T10:08:06.738979+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:32.280697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.680371+0000","last_change":"2024-09-21T10:03:53.840992+0000","last_active":"2024-09-21T10:07:59.680371+0000","last_peered":"2024-09-21T10:07:59.680371+0000","last_clean":"2024-09-21T10:07:59.680371+0000","last_became_active":"2024-09-21T10:03:53.840551+0000","last_became_peered":"2024-09-21T10:03:53.840551+0000","last_unstale":"2024-09-21T10:07:59.680371+0000","last_undegraded":"2024-09-21T10:07:59.680371+0000","last_fullsized":"2024-09-21T10:07:59.680371+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:04.442126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":100,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.680585+0000","last_change":"2024-09-21T10:02:46.659513+0000","last_active":"2024-09-21T10:07:59.680585+0000","last_peered":"2024-09-21T10:07:59.680585+0000","last_clean":"2024-09-21T10:07:59.680585+0000","last_became_active":"2024-09-21T10:02:46.659130+0000","last_became_peered":"2024-09-21T10:02:46.659130+0000","last_unstale":"2024-09-21T10:07:59.680585+0000","last_undegraded":"2024-09-21T10:07:59.680585+0000","last_fullsized":"2024-09-21T10:07:59.680585+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:24.792175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.681567+0000","last_change":"2024-09-21T10:03:53.835006+0000","last_active":"2024-09-21T10:07:59.681567+0000","last_peered":"2024-09-21T10:07:59.681567+0000","last_clean":"2024-09-21T10:07:59.681567+0000","last_became_active":"2024-09-21T10:03:53.834608+0000","last_became_peered":"2024-09-21T10:03:53.834608+0000","last_unstale":"2024-09-21T10:07:59.681567+0000","last_undegraded":"2024-09-21T10:07:59.681567+0000","last_fullsized":"2024-09-21T10:07:59.681567+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:56.108606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.681902+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:07:59.681902+0000","last_peered":"2024-09-21T10:07:59.681902+0000","last_clean":"2024-09-21T10:07:59.681902+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:07:59.681902+0000","last_undegraded":"2024-09-21T10:07:59.681902+0000","last_fullsized":"2024-09-21T10:07:59.681902+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":143,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724576+0000","last_change":"2024-09-21T10:03:53.844043+0000","last_active":"2024-09-21T10:08:06.724576+0000","last_peered":"2024-09-21T10:08:06.724576+0000","last_clean":"2024-09-21T10:08:06.724576+0000","last_became_active":"2024-09-21T10:03:53.842891+0000","last_became_peered":"2024-09-21T10:03:53.842891+0000","last_unstale":"2024-09-21T10:08:06.724576+0000","last_undegraded":"2024-09-21T10:08:06.724576+0000","last_fullsized":"2024-09-21T10:08:06.724576+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:39.891977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":132,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.724907+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:08:06.724907+0000","last_peered":"2024-09-21T10:08:06.724907+0000","last_clean":"2024-09-21T10:08:06.724907+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:08:06.724907+0000","last_undegraded":"2024-09-21T10:08:06.724907+0000","last_fullsized":"2024-09-21T10:08:06.724907+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-21T10:07:59.680254+0000","last_change":"2024-09-21T10:02:46.659818+0000","last_active":"2024-09-21T10:07:59.680254+0000","last_peered":"2024-09-21T10:07:59.680254+0000","last_clean":"2024-09-21T10:07:59.680254+0000","last_became_active":"2024-09-21T10:02:46.659457+0000","last_became_peered":"2024-09-21T10:02:46.659457+0000","last_unstale":"2024-09-21T10:07:59.680254+0000","last_undegraded":"2024-09-21T10:07:59.680254+0000","last_fullsized":"2024-09-21T10:07:59.680254+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:21.962330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":119,"reported_epoch":134,"state":"active+clean","last_fresh":"2024-09-21T10:08:06.738438+0000","last_change":"2024-09-21T10:03:19.769549+0000","last_active":"2024-09-21T10:08:06.738438+0000","last_peered":"2024-09-21T10:08:06.738438+0000","last_clean":"2024-09-21T10:08:06.738438+0000","last_became_active":"2024-09-21T10:03:19.768854+0000","last_became_peered":"2024-09-21T10:03:19.768854+0000","last_unstale":"2024-09-21T10:08:06.738438+0000","last_undegraded":"2024-09-21T10:08:06.738438+0000","last_fullsized":"2024-09-21T10:08:06.738438+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:29.415039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":4845731864,"num_objects":73941,"num_object_clones":0,"num_object_copies":221823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73941,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":192837,"num_write_kb":8536833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14485827584,"data_stored":14485815368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146837,"ondisk_log_size":146837,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234222,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1612076,"kb_used_data":1504672,"kb_used_omap":1,"kb_used_meta":107390,"kb_avail":92133076,"statfs":{"total":95995035648,"available":94344269824,"internally_reserved":0,"allocated":1540784128,"data_stored":1540075724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":109967819},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495860,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1256224,"kb_used_data":1189016,"kb_used_omap":1,"kb_used_meta":67198,"kb_avail":92488928,"statfs":{"total":95995035648,"available":94708662272,"internally_reserved":0,"allocated":1217552384,"data_stored":1216852352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":68811207},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757497,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2815000,"kb_used_data":2671440,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":90930152,"statfs":{"total":95995035648,"available":93112475648,"internally_reserved":0,"allocated":2735554560,"data_stored":2734858026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":146930118},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019133,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2540768,"kb_used_data":2393176,"kb_used_omap":1,"kb_used_meta":147582,"kb_avail":91204384,"statfs":{"total":95995035648,"available":93393289216,"internally_reserved":0,"allocated":2450612224,"data_stored":2449911705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":151124422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435318,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":950028,"kb_used_data":873924,"kb_used_omap":5,"kb_used_meta":76090,"kb_avail":92795124,"statfs":{"total":95995035648,"available":95022206976,"internally_reserved":0,"allocated":894898176,"data_stored":894213337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":77916200},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598844,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1916944,"kb_used_data":1804744,"kb_used_omap":7,"kb_used_meta":112184,"kb_avail":91828208,"statfs":{"total":95995035648,"available":94032084992,"internally_reserved":0,"allocated":1848057856,"data_stored":1847369163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":114876877},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762371,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1590476,"kb_used_data":1496320,"kb_used_omap":7,"kb_used_meta":94136,"kb_avail":92154676,"statfs":{"total":95995035648,"available":94366388224,"internally_reserved":0,"allocated":1532231680,"data_stored":1531550593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":96395517},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925898,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2456276,"kb_used_data":2312140,"kb_used_omap":7,"kb_used_meta":144120,"kb_avail":91288876,"statfs":{"total":95995035648,"available":93479809024,"internally_reserved":0,"allocated":2367631360,"data_stored":2366939387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":147579070},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,4,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2359099392,"data_stored":2359099392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1511194624,"data_stored":1511194624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1844514816,"data_stored":1844510744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":892534784,"data_stored":892534784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2443182080,"data_stored":2443182080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2710048768,"data_stored":2710044696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1208287232,"data_stored":1208287232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1516965888,"data_stored":1516961816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:08:16.418 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:08:16.419 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_0 pg_num 26 2024-09-21T10:08:16.438 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:16.539 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:16.573 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:16.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:16.674 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.723 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:16.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:16.824 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:16.863 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:16.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:16.965 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:17.009 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:17.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:17.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:17.150 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:17.252 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:17.280 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:17.281 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:17.289 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:17.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:17.391 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:17.429 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:17.531 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:17.560 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:17.662 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:17.699 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:17.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:17.801 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:17.840 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:17.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:17.942 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:17.972 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:17.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:17 smithi045 ceph-mon[95976]: pgmap v406: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-09-21T10:08:18.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:18.114 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:18.157 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:17 smithi032 ceph-mon[105748]: pgmap v406: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-09-21T10:08:18.158 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:17 smithi032 ceph-mon[101299]: pgmap v406: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-09-21T10:08:18.216 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:18.248 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:18.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:18.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.386 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:18.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:18.487 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:18.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:18.627 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:18.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.660 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:18.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:18.761 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:18.794 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:18.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:18.896 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:18.929 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:18.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:19.030 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:19.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.078 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:19.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:19.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:19.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:19.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:19.356 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:19.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:19.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:19.490 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:19.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.510 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:19.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:19.611 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:19.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:19.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:19.748 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:19.779 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:19.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:19.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:19 smithi032 ceph-mon[105748]: pgmap v407: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2024-09-21T10:08:19.840 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:19 smithi032 ceph-mon[101299]: pgmap v407: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2024-09-21T10:08:19.840 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:08:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:08:19] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.51.0" 2024-09-21T10:08:19.881 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:19.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:19.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:20.020 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:20.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:19 smithi045 ceph-mon[95976]: pgmap v407: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2024-09-21T10:08:20.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.058 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:20.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:20.159 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:20.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:20.303 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:20.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:20.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:20.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:20.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.470 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:20.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:20.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:20.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:20.708 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:20.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:20.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:20.855 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:20.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:21.006 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:21.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.050 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:21.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:21.152 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:21.179 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:21.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:21.281 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:21.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:21.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:21.429 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:21.466 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:21.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:21.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:21.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:21.706 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:21.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:21.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:21.790 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:08:21.846 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:21.883 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:21 smithi032 ceph-mon[101299]: pgmap v408: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-09-21T10:08:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:21 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/63183492' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:21 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:21 smithi032 ceph-mon[105748]: pgmap v408: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-09-21T10:08:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:21 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/63183492' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:21 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:21.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:22.018 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:22.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:21 smithi045 ceph-mon[95976]: pgmap v408: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-09-21T10:08:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:21 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/63183492' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:21 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:22.119 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:22.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.164 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:22.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:22.265 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:22.312 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:22.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:22.417 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:22.453 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:22.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:22.555 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:22.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:22.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:22.691 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:22.734 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:22.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:22.836 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:22.874 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:22.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:22.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:22 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:22.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:22 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:08:22.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:22 smithi032 ceph-mon[105748]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T10:08:22.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:22 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:22.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:22 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:08:22.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:22 smithi032 ceph-mon[101299]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T10:08:22.976 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:23.016 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:23.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:23.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:22 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:22 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:08:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:22 smithi045 ceph-mon[95976]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T10:08:23.117 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:23.155 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:23.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:23.232 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:23.232 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:23.256 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:23.293 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:23.395 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:23.427 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:23.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:23.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:23.570 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:23.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:23.671 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:23.717 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:23.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:23.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:23.862 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:23.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:23.871 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:23 smithi032 ceph-mon[101299]: pgmap v410: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2024-09-21T10:08:23.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:23 smithi032 ceph-mon[105748]: pgmap v410: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2024-09-21T10:08:23.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:08:23.965 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:24.004 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:24.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:23 smithi045 ceph-mon[95976]: pgmap v410: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2024-09-21T10:08:24.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:24.144 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:24.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:24.277 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:24.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:24.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.427 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:24.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:24.528 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:24.568 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:24.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:24.670 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:24.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:08:24.712 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:24.814 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:24.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:24.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:24.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:24.976 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:24.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:25.078 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:25.091 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:25.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:25.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:25.212 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.253 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:25.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:25.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:25.391 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:25.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:25.493 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:25.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.531 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:25.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:25.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:25.680 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:25.782 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:25.812 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 5 pg_num to 26 2024-09-21T10:08:25.824 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:25.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:25.831 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:25 smithi032 ceph-mon[101299]: pgmap v411: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-09-21T10:08:25.831 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1842883418' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:08:25.831 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:25 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:08:25.832 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:25 smithi032 ceph-mon[105748]: pgmap v411: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-09-21T10:08:25.832 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1842883418' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:08:25.832 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:25 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:08:25.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:25.968 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:25.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:26.069 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:26.106 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:26.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:26.207 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:26.249 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:26.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:25 smithi045 ceph-mon[95976]: pgmap v411: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-09-21T10:08:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1842883418' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:08:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:25 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:08:26.350 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:26.390 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:26.491 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:26.532 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:26.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:26.634 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:26.664 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:26.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:26.766 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:26.790 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:08:26.790 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:26.797 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:26.834 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:08:26.835 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:26.889 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:08:26.889 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:26.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:26 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:08:26.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:26 smithi032 ceph-mon[101299]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T10:08:26.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:26 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:08:26.890 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1244917840' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:26.890 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:26 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:08:26.890 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:26 smithi032 ceph-mon[105748]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T10:08:26.891 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:26 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:08:26.891 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1244917840' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:26.899 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:26.924 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:08:26.925 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:26.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:26.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:26.967 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:08:26.967 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:27.007 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:08:27.008 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:27.043 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:08:27.043 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:27.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:27.082 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:08:27.082 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:27.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.090 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:27.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:27.191 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:27.233 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:26 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:08:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:26 smithi045 ceph-mon[95976]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T10:08:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:26 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:08:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1244917840' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:27.335 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:27.358 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:27.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:27.459 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:27.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:27.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:27.583 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:27.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:27.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:27.706 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:27.728 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:27.822 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:27.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:27.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:27.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:27.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:28.020 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:28.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:28.077 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:27 smithi032 ceph-mon[105748]: pgmap v413: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-21T10:08:28.077 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:27 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:08:28.077 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:27 smithi032 ceph-mon[105748]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T10:08:28.077 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:27 smithi032 ceph-mon[101299]: pgmap v413: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-21T10:08:28.078 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:27 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:08:28.078 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:27 smithi032 ceph-mon[101299]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T10:08:28.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:28.152 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:28.253 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:27 smithi045 ceph-mon[95976]: pgmap v413: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-21T10:08:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:27 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:08:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:27 smithi045 ceph-mon[95976]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T10:08:28.291 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:28.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:28.423 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:28.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:28.525 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:28.558 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:28.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:28.660 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.698 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:28.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:28.800 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:28.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.836 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:28.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:28.937 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:28.944 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:08:28.944 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:08:28.944 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:08:28.944 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:08:28.945 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:08:28.974 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:28.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:29.076 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:29.115 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:29.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:29.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:28 smithi032 ceph-mon[105748]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T10:08:29.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:28 smithi032 ceph-mon[105748]: 5.4 scrub starts 2024-09-21T10:08:29.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:28 smithi032 ceph-mon[105748]: 5.6 scrub starts 2024-09-21T10:08:29.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:28 smithi032 ceph-mon[101299]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T10:08:29.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:28 smithi032 ceph-mon[101299]: 5.4 scrub starts 2024-09-21T10:08:29.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:28 smithi032 ceph-mon[101299]: 5.6 scrub starts 2024-09-21T10:08:29.217 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:29.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.252 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:29.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:29.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:28 smithi045 ceph-mon[95976]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T10:08:29.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:28 smithi045 ceph-mon[95976]: 5.4 scrub starts 2024-09-21T10:08:29.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:28 smithi045 ceph-mon[95976]: 5.6 scrub starts 2024-09-21T10:08:29.353 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:29.385 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:29.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:29.486 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:29.518 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:29.620 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:29.657 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:29.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:29.758 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:29.782 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:08:29.799 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:29.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:29.810 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:08:29] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.51.0" 2024-09-21T10:08:29.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:08:29.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:29.975 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:29.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:30.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[101299]: pgmap v416: 99 pgs: 10 unknown, 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-21T10:08:30.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[101299]: 5.4 scrub ok 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[101299]: 5.0 scrub starts 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[101299]: 5.6 scrub ok 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[101299]: 5.9 scrub starts 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[101299]: 5.8 scrub starts 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[101299]: 5.9 scrub ok 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[105748]: pgmap v416: 99 pgs: 10 unknown, 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[105748]: 5.4 scrub ok 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[105748]: 5.0 scrub starts 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[105748]: 5.6 scrub ok 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[105748]: 5.9 scrub starts 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[105748]: 5.8 scrub starts 2024-09-21T10:08:30.071 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:29 smithi032 ceph-mon[105748]: 5.9 scrub ok 2024-09-21T10:08:30.077 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:30.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:30.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:30.212 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:30.250 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:30.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:29 smithi045 ceph-mon[95976]: pgmap v416: 99 pgs: 10 unknown, 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-21T10:08:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:29 smithi045 ceph-mon[95976]: 5.4 scrub ok 2024-09-21T10:08:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:29 smithi045 ceph-mon[95976]: 5.0 scrub starts 2024-09-21T10:08:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:29 smithi045 ceph-mon[95976]: 5.6 scrub ok 2024-09-21T10:08:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:29 smithi045 ceph-mon[95976]: 5.9 scrub starts 2024-09-21T10:08:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:29 smithi045 ceph-mon[95976]: 5.8 scrub starts 2024-09-21T10:08:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:29 smithi045 ceph-mon[95976]: 5.9 scrub ok 2024-09-21T10:08:30.352 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:30.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.387 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:30.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:30.488 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:30.529 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:30.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:30.631 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:30.670 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:30.771 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:30.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:30.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:30.913 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:30.947 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:30.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:30 smithi032 ceph-mon[101299]: 5.0 scrub ok 2024-09-21T10:08:30.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:30 smithi032 ceph-mon[101299]: 5.7 scrub starts 2024-09-21T10:08:30.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:30 smithi032 ceph-mon[101299]: 5.8 scrub ok 2024-09-21T10:08:30.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:30 smithi032 ceph-mon[105748]: 5.0 scrub ok 2024-09-21T10:08:30.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:30 smithi032 ceph-mon[105748]: 5.7 scrub starts 2024-09-21T10:08:30.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:30 smithi032 ceph-mon[105748]: 5.8 scrub ok 2024-09-21T10:08:31.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:31.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:31.197 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:31.232 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:31.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:30 smithi045 ceph-mon[95976]: 5.0 scrub ok 2024-09-21T10:08:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:30 smithi045 ceph-mon[95976]: 5.7 scrub starts 2024-09-21T10:08:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:30 smithi045 ceph-mon[95976]: 5.8 scrub ok 2024-09-21T10:08:31.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:31.368 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:31.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:31.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:31.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:31.612 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:31.659 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:31.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:31.761 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:31.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:31.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:31.893 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:31.897 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:31.924 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:31.924 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:32.026 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:32.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.064 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:32.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:32.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:32.204 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:32.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:32.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[105748]: pgmap v417: 99 pgs: 1 active+clean+scrubbing, 3 unknown, 95 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[105748]: 5.7 scrub ok 2024-09-21T10:08:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[105748]: 5.10 scrub starts 2024-09-21T10:08:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[105748]: 5.17 scrub starts 2024-09-21T10:08:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[105748]: 5.10 scrub ok 2024-09-21T10:08:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[101299]: pgmap v417: 99 pgs: 1 active+clean+scrubbing, 3 unknown, 95 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[101299]: 5.7 scrub ok 2024-09-21T10:08:32.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[101299]: 5.10 scrub starts 2024-09-21T10:08:32.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[101299]: 5.17 scrub starts 2024-09-21T10:08:32.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:31 smithi032 ceph-mon[101299]: 5.10 scrub ok 2024-09-21T10:08:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:31 smithi045 ceph-mon[95976]: pgmap v417: 99 pgs: 1 active+clean+scrubbing, 3 unknown, 95 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:31 smithi045 ceph-mon[95976]: 5.7 scrub ok 2024-09-21T10:08:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:31 smithi045 ceph-mon[95976]: 5.10 scrub starts 2024-09-21T10:08:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:31 smithi045 ceph-mon[95976]: 5.17 scrub starts 2024-09-21T10:08:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:31 smithi045 ceph-mon[95976]: 5.10 scrub ok 2024-09-21T10:08:32.306 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:32.347 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:32.448 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:32.482 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:32.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:32.584 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:32.625 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:32.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:32.727 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:32.763 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:32.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:32.865 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:32.905 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:32.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:33.007 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:33.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.040 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:33.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:33.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:33.170 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:33.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[105748]: 5.1 scrub starts 2024-09-21T10:08:33.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[105748]: 5.17 scrub ok 2024-09-21T10:08:33.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[105748]: 5.1 scrub ok 2024-09-21T10:08:33.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:08:33.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[105748]: 5.16 scrub starts 2024-09-21T10:08:33.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[101299]: 5.1 scrub starts 2024-09-21T10:08:33.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[101299]: 5.17 scrub ok 2024-09-21T10:08:33.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[101299]: 5.1 scrub ok 2024-09-21T10:08:33.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:08:33.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:32 smithi032 ceph-mon[101299]: 5.16 scrub starts 2024-09-21T10:08:33.271 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:32 smithi045 ceph-mon[95976]: 5.1 scrub starts 2024-09-21T10:08:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:32 smithi045 ceph-mon[95976]: 5.17 scrub ok 2024-09-21T10:08:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:32 smithi045 ceph-mon[95976]: 5.1 scrub ok 2024-09-21T10:08:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:32 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:08:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:32 smithi045 ceph-mon[95976]: 5.16 scrub starts 2024-09-21T10:08:33.297 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:33.399 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:33.422 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:33.524 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:33.545 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:33.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:33.647 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:33.672 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:33.773 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:33.795 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:33.897 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:33.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[101299]: pgmap v418: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:08:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[101299]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T10:08:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[101299]: 5.16 scrub ok 2024-09-21T10:08:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[101299]: 5.19 scrub starts 2024-09-21T10:08:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[101299]: 5.19 scrub ok 2024-09-21T10:08:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[105748]: pgmap v418: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:08:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[105748]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T10:08:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[105748]: 5.16 scrub ok 2024-09-21T10:08:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[105748]: 5.19 scrub starts 2024-09-21T10:08:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[105748]: 5.19 scrub ok 2024-09-21T10:08:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:34.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:34.046 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:34.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:34.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:34.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:34.272 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:33 smithi045 ceph-mon[95976]: pgmap v418: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:08:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:33 smithi045 ceph-mon[95976]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T10:08:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:33 smithi045 ceph-mon[95976]: 5.16 scrub ok 2024-09-21T10:08:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:33 smithi045 ceph-mon[95976]: 5.19 scrub starts 2024-09-21T10:08:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:33 smithi045 ceph-mon[95976]: 5.19 scrub ok 2024-09-21T10:08:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:34.296 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:34.342 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:08:34.343 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:08:34.343 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-09-21T10:08:34.343 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:08:34.343 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:08:34.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-09-21T10:08:34.399 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:34.423 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:34.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:34.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:34.551 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:34.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:34.653 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:34.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:34.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:34.776 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:34.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:34.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:34.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:34.927 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:34.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:35.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:35.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.065 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:35.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:35.166 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:35.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:35.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[105748]: 5.2 scrub starts 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[105748]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[105748]: 5.2 scrub ok 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[105748]: 5.18 scrub starts 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[105748]: 5.18 scrub ok 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[101299]: 5.2 scrub starts 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[101299]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[101299]: 5.2 scrub ok 2024-09-21T10:08:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:08:35.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[101299]: 5.18 scrub starts 2024-09-21T10:08:35.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:34 smithi032 ceph-mon[101299]: 5.18 scrub ok 2024-09-21T10:08:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:34 smithi045 ceph-mon[95976]: 5.2 scrub starts 2024-09-21T10:08:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:34 smithi045 ceph-mon[95976]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T10:08:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:34 smithi045 ceph-mon[95976]: 5.2 scrub ok 2024-09-21T10:08:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:08:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:34 smithi045 ceph-mon[95976]: 5.18 scrub starts 2024-09-21T10:08:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:34 smithi045 ceph-mon[95976]: 5.18 scrub ok 2024-09-21T10:08:35.302 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:35.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:35.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:35.438 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:35.465 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:35.567 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:35.592 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:35.640 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:08:35.640 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:08:35.640 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:26448 5.5 5:a8a2e575:::benchmark_data_smithi032_140417_object26447:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:05:38.496067+0000", 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 177.14020924900001, 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16593353199999999, 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26448 2024-09-21T10:08:35.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496067+0000", 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496067+0000", 2024-09-21T10:08:35.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496068+0000", 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5710000000000001e-06 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496090+0000", 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1607000000000002e-05 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496111+0000", 2024-09-21T10:08:35.643 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1115e-05 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496119+0000", 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8990000000000001e-06 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496186+0000", 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7448999999999997e-05 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496323+0000", 2024-09-21T10:08:35.644 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013674 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.496510+0000", 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000187142 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.497646+0000", 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0011356649999999999 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.645 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.498639+0000", 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00099301100000000002 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.660460+0000", 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.161821091 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.660518+0000", 2024-09-21T10:08:35.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8563999999999997e-05 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:05:38.662000+0000", 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014816779999999999 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:90186 5.5 5:a3bedc52:::benchmark_data_smithi032_140417_object90185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:18.396716+0000", 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 137.239560278, 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13699291099999999, 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 90186 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.396716+0000", 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.648 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.396716+0000", 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.396717+0000", 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4789999999999999e-06 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.396761+0000", 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3640999999999998e-05 2024-09-21T10:08:35.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.396772+0000", 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1421e-05 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.396780+0000", 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7640000000000005e-06 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.400111+0000", 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0033315490000000001 2024-09-21T10:08:35.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.400258+0000", 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014621799999999999 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.400679+0000", 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00042167099999999998 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.403942+0000", 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0032624239999999999 2024-09-21T10:08:35.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.533600+0000", 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12965839400000001 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.533641+0000", 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1233999999999999e-05 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.533676+0000", 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4724999999999997e-05 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:18.533708+0000", 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2391000000000002e-05 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:99167 5.b e124/112 5:d3dc5508:::benchmark_data_smithi032_140417_object99166:head v 124'6201, mlcod=124'6198)", 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.270331+0000", 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 129.36594468800001, 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14457615700000001, 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.653 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.654 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.654 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270331+0000", 2024-09-21T10:08:35.654 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270331+0000", 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270332+0000", 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2099999999999995e-07 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.655 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270361+0000", 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.9130999999999999e-05 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270372+0000", 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0499000000000001e-05 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270378+0000", 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6780000000000003e-06 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270418+0000", 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9988999999999999e-05 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270431+0000", 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2812e-05 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.414861+0000", 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14443019100000001 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.414907+0000", 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6035999999999998e-05 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:99168 5.7 e124/112 5:e036d327:::benchmark_data_smithi032_140417_object99167:head v 124'6300, mlcod=124'6298)", 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.270773+0000", 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 129.365502808, 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.146213285, 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.658 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270773+0000", 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270773+0000", 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.659 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270774+0000", 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6600000000000005e-07 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270831+0000", 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6855999999999999e-05 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270849+0000", 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7861000000000001e-05 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270855+0000", 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3759999999999996e-06 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270911+0000", 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5733999999999999e-05 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.270930+0000", 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8975999999999999e-05 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.411678+0000", 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14074790300000001 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.416986+0000", 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0053087129999999996 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:99174 5.c e124/112 5:3acaa447:::benchmark_data_smithi032_140417_object99173:head v 124'6124, mlcod=124'6121)", 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.273498+0000", 2024-09-21T10:08:35.662 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 129.36277830500001, 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14204820500000001, 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.273498+0000", 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.273498+0000", 2024-09-21T10:08:35.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.273498+0000", 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0800000000000003e-07 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.273521+0000", 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2736000000000001e-05 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.273531+0000", 2024-09-21T10:08:35.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0226e-05 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.273545+0000", 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3174e-05 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.273605+0000", 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0690999999999999e-05 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.273624+0000", 2024-09-21T10:08:35.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8471999999999999e-05 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276867+0000", 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0032434130000000001 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.415546+0000", 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13867858499999999 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:99172 5.7 e124/112 5:efc380b5:::benchmark_data_smithi032_140417_object99171:head v 124'6301, mlcod=124'6298)", 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.274236+0000", 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 129.362039409, 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14558412200000001, 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274236+0000", 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274236+0000", 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274237+0000", 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8599999999999997e-07 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274280+0000", 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2469000000000003e-05 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274291+0000", 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1369000000000001e-05 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274297+0000", 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3729999999999998e-06 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274368+0000", 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0365999999999997e-05 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.274385+0000", 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7680999999999999e-05 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.419779+0000", 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14539374599999999 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.419821+0000", 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1331999999999997e-05 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.670 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:99169 5.b e124/112 5:d6a9cf4a:::benchmark_data_smithi032_140417_object99168:head v 124'6202, mlcod=124'6198)", 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.275478+0000", 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 129.36079743499999, 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.15381926200000001, 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275478+0000", 2024-09-21T10:08:35.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275478+0000", 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275479+0000", 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7599999999999996e-07 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275504+0000", 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4606000000000001e-05 2024-09-21T10:08:35.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275515+0000", 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0933e-05 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275525+0000", 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0604999999999999e-05 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275603+0000", 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7476000000000002e-05 2024-09-21T10:08:35.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.275627+0000", 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4162e-05 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.429250+0000", 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.153623018 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.429298+0000", 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7586000000000001e-05 2024-09-21T10:08:35.674 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:99177 5.5 5:adba63d9:::benchmark_data_smithi032_140417_object99176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.276550+0000", 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 129.35972534800001, 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.145295551, 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.675 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 99177 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276550+0000", 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276550+0000", 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276552+0000", 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.652e-06 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276580+0000", 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.7807999999999999e-05 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276593+0000", 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.256e-05 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276600+0000", 2024-09-21T10:08:35.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6529999999999994e-06 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276681+0000", 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0612999999999995e-05 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276842+0000", 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000161643 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412580+0000", 2024-09-21T10:08:35.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.135737365 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.421270+0000", 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0086902799999999999 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.421324+0000", 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3888000000000003e-05 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.421740+0000", 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00041586900000000003 2024-09-21T10:08:35.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.421794+0000", 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3822000000000001e-05 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.421846+0000", 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2398000000000002e-05 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:99178 5.e e124/112 5:7a3c961f:::benchmark_data_smithi032_140417_object99177:head v 124'6189, mlcod=124'6187)", 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.276637+0000", 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 129.359639114, 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13577256400000001, 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276637+0000", 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.681 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276637+0000", 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276637+0000", 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5600000000000005e-07 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276668+0000", 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0701000000000003e-05 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276676+0000", 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.242e-06 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276682+0000", 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5470000000000004e-06 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276730+0000", 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7945999999999998e-05 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.276748+0000", 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7903000000000001e-05 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412324+0000", 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13557604600000001 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412409+0000", 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5422999999999998e-05 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.684 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:99182 5.c e124/112 5:3705e000:::benchmark_data_smithi032_140417_object99181:head v 124'6125, mlcod=124'6122)", 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.279223+0000", 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 129.35705250500001, 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14266235199999999, 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.279223+0000", 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.279223+0000", 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.279225+0000", 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1680000000000001e-06 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.279249+0000", 2024-09-21T10:08:35.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4695e-05 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412510+0000", 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13326112700000001 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.412521+0000", 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1167e-05 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.687 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.415584+0000", 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0030622729999999999 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.415605+0000", 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0752000000000001e-05 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.421811+0000", 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0062065710000000001 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.688 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.421886+0000", 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4598999999999995e-05 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:99183 5.3 5:c85d0ec1:::benchmark_data_smithi032_140417_object99182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:06:26.281173+0000", 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 129.35510267199999, 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.15287608899999999, 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.689 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 99183 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.281173+0000", 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.281173+0000", 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.281175+0000", 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2539999999999999e-06 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.281199+0000", 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3289e-05 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.411618+0000", 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.13041968800000001 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.411650+0000", 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1208999999999999e-05 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.417739+0000", 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.006089662 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.417861+0000", 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000122139 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.692 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.417996+0000", 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013501600000000001 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.422824+0000", 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0048275940000000002 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.693 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.694 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.433905+0000", 2024-09-21T10:08:35.694 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011081369000000001 2024-09-21T10:08:35.694 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.694 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.433936+0000", 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0932000000000001e-05 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.433995+0000", 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8972999999999999e-05 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:06:26.434049+0000", 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3964e-05 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207613 5.3 5:c7f09cfd:::benchmark_data_smithi032_140417_object74222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:26.810342+0000", 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.8259341720000002, 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.39366306499999998, 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207613 2024-09-21T10:08:35.696 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.810342+0000", 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.810342+0000", 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.810344+0000", 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0600000000000002e-06 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.810350+0000", 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2670000000000003e-06 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.810369+0000", 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9103e-05 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.810378+0000", 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7020000000000007e-06 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.810541+0000", 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000163394 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.810898+0000", 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000356776 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.699 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.815558+0000", 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0046599830000000004 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.825948+0000", 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010390117000000001 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.203988+0000", 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.37804012999999997 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.204005+0000", 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6532999999999999e-05 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:207616 5.8 e136/112 5:130c89fe:::benchmark_data_smithi032_140417_object74225:head v 136'12923, mlcod=136'12921)", 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:26.811168+0000", 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.8251076449999992, 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.422696719, 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.701 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.811168+0000", 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.811168+0000", 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.811169+0000", 2024-09-21T10:08:35.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.263e-06 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.811177+0000", 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2439999999999996e-06 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.811195+0000", 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8261e-05 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.811206+0000", 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1222e-05 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.811944+0000", 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000738093 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.811963+0000", 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8439000000000001e-05 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.233865+0000", 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.42190219699999998 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:207619 5.8 e136/112 5:1f63d48b:::benchmark_data_smithi032_140417_object74228:head v 136'12924, mlcod=136'12921)", 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:26.820283+0000", 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.8159929689999998, 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.41371886699999999, 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:08:35.705 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.820283+0000", 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.820283+0000", 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.820284+0000", 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1060000000000001e-06 2024-09-21T10:08:35.706 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.820292+0000", 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8949999999999998e-06 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.820304+0000", 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2439999999999999e-05 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.820312+0000", 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6510000000000001e-06 2024-09-21T10:08:35.707 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828407+0000", 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0080948979999999997 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828429+0000", 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1800000000000001e-05 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.234002+0000", 2024-09-21T10:08:35.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.405573077 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207623 5.3 5:c041238e:::benchmark_data_smithi032_140417_object74232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:26.828037+0000", 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.8082390490000009, 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.376012503, 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.709 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207623 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828037+0000", 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828037+0000", 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828039+0000", 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5009999999999999e-06 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828049+0000", 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.9920000000000002e-06 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828073+0000", 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3917000000000002e-05 2024-09-21T10:08:35.711 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828083+0000", 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8649999999999996e-06 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.831647+0000", 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003564139 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.831784+0000", 2024-09-21T10:08:35.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013685699999999999 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.831904+0000", 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000119503 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.204040+0000", 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.37213654200000001 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.204049+0000", 2024-09-21T10:08:35.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1870000000000003e-06 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44543.0:207617 5.b e136/112 5:d620d0cb:::benchmark_data_smithi032_140417_object74226:head v 136'12959, mlcod=136'12956)", 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:26.828141+0000", 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.8081344510000008, 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.37471843799999999, 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828141+0000", 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828141+0000", 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828142+0000", 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.28e-07 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828147+0000", 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0019999999999997e-06 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828155+0000", 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3580000000000003e-06 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828161+0000", 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7780000000000004e-06 2024-09-21T10:08:35.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828767+0000", 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00060674000000000004 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.828787+0000", 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9502999999999999e-05 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.202790+0000", 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.37400291800000002 2024-09-21T10:08:35.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.202860+0000", 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9911000000000002e-05 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207627 5.5 5:a8f2311e:::benchmark_data_smithi032_140417_object74236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:26.837288+0000", 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.7989875990000002, 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.367085938, 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.718 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207627 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.837288+0000", 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.837288+0000", 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.837291+0000", 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3590000000000002e-06 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.837299+0000", 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2190000000000004e-06 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.837317+0000", 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8094999999999999e-05 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.837329+0000", 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.187e-05 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.837413+0000", 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4300000000000003e-05 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:08:35.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.837551+0000", 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000137596 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.837676+0000", 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012491800000000001 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:26.850933+0000", 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013257592 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.204362+0000", 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.35342872800000003 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.204374+0000", 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2261e-05 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207613 5.3 5:c7f09cfd:::benchmark_data_smithi032_140417_object74222:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:27.041233+0000", 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.5950424109999997, 2024-09-21T10:08:35.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.94483936999999996, 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207613 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041233+0000", 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.724 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041233+0000", 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041236+0000", 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2419999999999999e-06 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041243+0000", 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2799999999999998e-06 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.725 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041264+0000", 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.124e-05 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041274+0000", 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0217999999999999e-05 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.205547+0000", 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16427294000000001 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.205560+0000", 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2215000000000001e-05 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.923146+0000", 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.71758636499999995 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for missing object", 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.923342+0000", 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00019561399999999999 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.986004+0000", 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.062662206999999998 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.986073+0000", 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9048999999999996e-05 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207623 5.3 5:c041238e:::benchmark_data_smithi032_140417_object74232:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2024-09-21T10:08:35.728 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:27.041291+0000", 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.5949847039999998, 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95519764399999996, 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207623 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041291+0000", 2024-09-21T10:08:35.729 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041291+0000", 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041292+0000", 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0999999999999999e-07 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041294+0000", 2024-09-21T10:08:35.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3479999999999998e-06 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041300+0000", 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1260000000000003e-06 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041305+0000", 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.169e-06 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.205592+0000", 2024-09-21T10:08:35.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16428699799999999 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.205602+0000", 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5179999999999993e-06 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.986098+0000", 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.78049586299999996 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for missing object", 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.986271+0000", 2024-09-21T10:08:35.732 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017311400000000001 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.996390+0000", 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010119655 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.996489+0000", 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8443000000000006e-05 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.733 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207627 5.15 5:a8f2311e:::benchmark_data_smithi032_140417_object74236:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:27.041312+0000", 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 8.5949638470000007, 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.87804795999999996, 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207627 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.734 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041312+0000", 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041312+0000", 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041312+0000", 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6199999999999999e-07 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:08:35.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041315+0000", 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2900000000000001e-06 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041321+0000", 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1489999999999999e-06 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.041326+0000", 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7389999999999999e-06 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.204214+0000", 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16288851600000001 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.204222+0000", 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4149999999999999e-06 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.919279+0000", 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.71505605900000002 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.919360+0000", 2024-09-21T10:08:35.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1429999999999998e-05 2024-09-21T10:08:35.738 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.738 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.738 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.738 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:08:35.738 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:08:35.738 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:08:35.738 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:35.763 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:35.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:35.864 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:35.887 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:08:35.892 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:35.994 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:36.018 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:36.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:36.119 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:36.144 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:36.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[105748]: pgmap v421: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:08:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[105748]: 5.10 starting backfill to osd.6 from (0'0,0'0] MAX to 138'13189 2024-09-21T10:08:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[105748]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T10:08:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[105748]: 5.10 starting backfill to osd.7 from (0'0,0'0] MAX to 138'13189 2024-09-21T10:08:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2700442447' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:36.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[101299]: pgmap v421: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:36.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:08:36.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[101299]: 5.10 starting backfill to osd.6 from (0'0,0'0] MAX to 138'13189 2024-09-21T10:08:36.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[101299]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T10:08:36.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[101299]: 5.10 starting backfill to osd.7 from (0'0,0'0] MAX to 138'13189 2024-09-21T10:08:36.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2700442447' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:36.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:35 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:36.245 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:36.270 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:36.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:35 smithi045 ceph-mon[95976]: pgmap v421: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:35 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:08:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:35 smithi045 ceph-mon[95976]: 5.10 starting backfill to osd.6 from (0'0,0'0] MAX to 138'13189 2024-09-21T10:08:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:35 smithi045 ceph-mon[95976]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T10:08:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:35 smithi045 ceph-mon[95976]: 5.10 starting backfill to osd.7 from (0'0,0'0] MAX to 138'13189 2024-09-21T10:08:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:35 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2700442447' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:35 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:36.372 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:36.396 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:36.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:36.497 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:36.521 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:36.623 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:36.646 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:36.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:36.748 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:36.772 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:36.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:36.873 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:36.901 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:37.003 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:37.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:37.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:37.142 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:37.179 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:37.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:08:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:08:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[105748]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T10:08:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[105748]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T10:08:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:08:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:08:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[101299]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T10:08:37.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:36 smithi032 ceph-mon[101299]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T10:08:37.281 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:36 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:08:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:36 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:08:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:36 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:36 smithi045 ceph-mon[95976]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T10:08:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:36 smithi045 ceph-mon[95976]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T10:08:37.309 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:37.411 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:37.443 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:37.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:37.544 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:37.574 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:37.676 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:37.713 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:37.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:37.815 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:37.853 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:37.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:37.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:37.994 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:38.096 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:38.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:37 smithi032 ceph-mon[101299]: pgmap v424: 99 pgs: 1 remapped+peering, 98 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-09-21T10:08:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:37 smithi032 ceph-mon[101299]: 5.11 starting backfill to osd.6 from (0'0,0'0] MAX to 139'12953 2024-09-21T10:08:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:37 smithi032 ceph-mon[105748]: pgmap v424: 99 pgs: 1 remapped+peering, 98 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-09-21T10:08:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:37 smithi032 ceph-mon[105748]: 5.11 starting backfill to osd.6 from (0'0,0'0] MAX to 139'12953 2024-09-21T10:08:38.233 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:38.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:38.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:38.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:37 smithi045 ceph-mon[95976]: pgmap v424: 99 pgs: 1 remapped+peering, 98 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-09-21T10:08:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:37 smithi045 ceph-mon[95976]: 5.11 starting backfill to osd.6 from (0'0,0'0] MAX to 139'12953 2024-09-21T10:08:38.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:38.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:38.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:38.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:38.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:38.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:38.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:38.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:38.781 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:38.803 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:38.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:38.905 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:38.939 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:38.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:39.041 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:39.078 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:39.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:39.180 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:38 smithi032 ceph-mon[101299]: 5.15 scrub starts 2024-09-21T10:08:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:38 smithi032 ceph-mon[105748]: 5.15 scrub starts 2024-09-21T10:08:39.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:39.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:38 smithi045 ceph-mon[95976]: 5.15 scrub starts 2024-09-21T10:08:39.321 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:39.354 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:39.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:39.456 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:39.496 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:39.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:39.598 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:39.638 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:39.740 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:39.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.780 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:39.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:39.810 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:08:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:08:39] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.51.0" 2024-09-21T10:08:39.882 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:39.914 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:39.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:40.016 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:40.054 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:40.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:40.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:39 smithi032 ceph-mon[101299]: pgmap v426: 99 pgs: 2 remapped+peering, 97 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-09-21T10:08:40.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:39 smithi032 ceph-mon[101299]: 5.15 scrub ok 2024-09-21T10:08:40.075 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:39 smithi032 ceph-mon[105748]: pgmap v426: 99 pgs: 2 remapped+peering, 97 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-09-21T10:08:40.075 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:39 smithi032 ceph-mon[105748]: 5.15 scrub ok 2024-09-21T10:08:40.156 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.192 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:40.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:39 smithi045 ceph-mon[95976]: pgmap v426: 99 pgs: 2 remapped+peering, 97 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-09-21T10:08:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:39 smithi045 ceph-mon[95976]: 5.15 scrub ok 2024-09-21T10:08:40.293 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:40.323 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:40.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:40.425 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:40.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:40.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:40.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:08:40.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:40.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.622 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:40.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:40.723 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:40.754 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:40.855 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:40.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.901 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:40.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:41.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:41.035 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:41.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:41.167 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:40 smithi032 ceph-mon[101299]: 5.5 scrub starts 2024-09-21T10:08:41.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:40 smithi032 ceph-mon[105748]: 5.5 scrub starts 2024-09-21T10:08:41.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:41.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:41.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:41.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:40 smithi045 ceph-mon[95976]: 5.5 scrub starts 2024-09-21T10:08:41.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:41.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:41.412 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:41.443 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:41.545 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:41.569 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:41.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:41.600 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:08:41.600 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:41.609 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:41.626 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:08:41.626 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:41.651 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:08:41.651 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:41.671 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:41.677 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:08:41.677 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:41.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.696 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:41.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:41.700 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:08:41.700 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:41.720 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:08:41.720 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:41.736 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:08:41.736 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:41.755 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:08:41.755 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:41.797 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:41.821 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:41.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:41.922 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:41.948 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:41.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:42.050 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:42.091 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:42.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:42.193 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:42.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:41 smithi032 ceph-mon[105748]: pgmap v427: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 404/152337 objects misplaced (0.265%); 14 MiB/s, 230 objects/s recovering 2024-09-21T10:08:42.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:41 smithi032 ceph-mon[105748]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T10:08:42.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:41 smithi032 ceph-mon[105748]: 5.5 scrub ok 2024-09-21T10:08:42.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:41 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2524801358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:42.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:41 smithi032 ceph-mon[101299]: pgmap v427: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 404/152337 objects misplaced (0.265%); 14 MiB/s, 230 objects/s recovering 2024-09-21T10:08:42.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:41 smithi032 ceph-mon[101299]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T10:08:42.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:41 smithi032 ceph-mon[101299]: 5.5 scrub ok 2024-09-21T10:08:42.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:41 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2524801358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:42.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:42.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:41 smithi045 ceph-mon[95976]: pgmap v427: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 404/152337 objects misplaced (0.265%); 14 MiB/s, 230 objects/s recovering 2024-09-21T10:08:42.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:41 smithi045 ceph-mon[95976]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T10:08:42.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:41 smithi045 ceph-mon[95976]: 5.5 scrub ok 2024-09-21T10:08:42.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:41 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2524801358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:42.330 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:42.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:42.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:42.465 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:42.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:42.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:42.600 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:42.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:42.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:42.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:42.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:42.889 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:42.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:43.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:43.067 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:43.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:43.168 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:43.188 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:42 smithi045 ceph-mon[95976]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T10:08:43.188 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:42 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:08:43.202 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:43.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:08:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:43.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:42 smithi032 ceph-mon[105748]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T10:08:43.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:08:43.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:42 smithi032 ceph-mon[101299]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T10:08:43.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:08:43.304 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:43.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:43.435 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:43.462 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:43.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:43.564 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:43.591 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:43.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:43.692 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:43.714 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:43.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:43.816 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:43.839 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:43.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:43.940 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:43.963 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:43.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:44.064 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:44.089 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:44.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:44.096 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:43 smithi032 ceph-mon[105748]: pgmap v430: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1577/149766 objects misplaced (1.053%); 18 MiB/s, 292 objects/s recovering 2024-09-21T10:08:44.097 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:43 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:08:44.097 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:43 smithi032 ceph-mon[105748]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T10:08:44.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:43 smithi032 ceph-mon[101299]: pgmap v430: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1577/149766 objects misplaced (1.053%); 18 MiB/s, 292 objects/s recovering 2024-09-21T10:08:44.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:43 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:08:44.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:43 smithi032 ceph-mon[101299]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T10:08:44.191 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:44.213 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:44.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:43 smithi045 ceph-mon[95976]: pgmap v430: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1577/149766 objects misplaced (1.053%); 18 MiB/s, 292 objects/s recovering 2024-09-21T10:08:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:43 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:08:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:43 smithi045 ceph-mon[95976]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T10:08:44.315 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:08:44.322 INFO:teuthology.orchestra.run.smithi032.stdout: "success": "" 2024-09-21T10:08:44.322 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:08:44.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:44.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:44.439 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:44.462 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:44.563 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:44.572 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:44.588 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:44.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:44.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:44.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:08:44.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.714 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:44.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:44.815 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:44.837 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:44.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:44.939 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:44.968 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:44.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:45.070 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:45.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:44 smithi032 ceph-mon[101299]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T10:08:45.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:44 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:08:45.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:44 smithi032 ceph-mon[101299]: 5.3 scrub starts 2024-09-21T10:08:45.084 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:44 smithi032 ceph-mon[105748]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T10:08:45.084 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:44 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:08:45.084 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:44 smithi032 ceph-mon[105748]: 5.3 scrub starts 2024-09-21T10:08:45.100 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:45.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:45.157 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:45.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:45.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.241 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:45.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:44 smithi045 ceph-mon[95976]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T10:08:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:44 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:08:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:44 smithi045 ceph-mon[95976]: 5.3 scrub starts 2024-09-21T10:08:45.342 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:45.368 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:45.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:45.470 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:45.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:45.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:45.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:45.625 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:45.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:45.727 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:45.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:45.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:45.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:45.883 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:45.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:45.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:46.021 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:46.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:46.154 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:46.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:46.166 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: pgmap v433: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1577/145233 objects misplaced (1.086%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T10:08:46.166 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: 5.11 scrub starts 2024-09-21T10:08:46.166 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: 5.3 scrub ok 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: 5.11 scrub ok 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: 5.12 starting backfill to osd.1 from (0'0,0'0] MAX to 145'13279 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: 5.12 starting backfill to osd.3 from (0'0,0'0] MAX to 145'13279 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: 5.14 scrub starts 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[101299]: 5.14 scrub ok 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: pgmap v433: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1577/145233 objects misplaced (1.086%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: 5.11 scrub starts 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: 5.3 scrub ok 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: 5.11 scrub ok 2024-09-21T10:08:46.167 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:08:46.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T10:08:46.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: 5.12 starting backfill to osd.1 from (0'0,0'0] MAX to 145'13279 2024-09-21T10:08:46.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: 5.12 starting backfill to osd.3 from (0'0,0'0] MAX to 145'13279 2024-09-21T10:08:46.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: 5.14 scrub starts 2024-09-21T10:08:46.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:45 smithi032 ceph-mon[105748]: 5.14 scrub ok 2024-09-21T10:08:46.256 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: pgmap v433: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1577/145233 objects misplaced (1.086%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: 5.11 scrub starts 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: 5.3 scrub ok 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: 5.11 scrub ok 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: 5.12 starting backfill to osd.1 from (0'0,0'0] MAX to 145'13279 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: 5.12 starting backfill to osd.3 from (0'0,0'0] MAX to 145'13279 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: 5.14 scrub starts 2024-09-21T10:08:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:45 smithi045 ceph-mon[95976]: 5.14 scrub ok 2024-09-21T10:08:46.287 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:46.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:46.389 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:46.412 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:46.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:46.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:46.536 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:46.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:46.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:46.665 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:46.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:46.766 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:46.793 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:46.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:46.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:46.921 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:47.023 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:47.052 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:47.154 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.187 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:47.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:46 smithi032 ceph-mon[105748]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T10:08:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:08:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:08:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:46 smithi032 ceph-mon[101299]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T10:08:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:08:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:08:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:46 smithi045 ceph-mon[95976]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T10:08:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:08:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:08:47.288 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:47.326 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:47.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:47.428 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:47.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:47.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:47.557 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:47.585 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:47.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:47.687 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:47.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:47.810 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:47.832 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:47.934 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:47.956 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:48.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:48.083 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:48.167 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[105748]: pgmap v436: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 1173/139650 objects misplaced (0.840%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[105748]: 5.11 scrub starts 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[105748]: 5.11 scrub ok 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[105748]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[105748]: 5.13 starting backfill to osd.1 from (0'0,0'0] MAX to 146'13431 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[105748]: 5.13 starting backfill to osd.3 from (0'0,0'0] MAX to 146'13431 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[101299]: pgmap v436: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 1173/139650 objects misplaced (0.840%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[101299]: 5.11 scrub starts 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[101299]: 5.11 scrub ok 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[101299]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[101299]: 5.13 starting backfill to osd.1 from (0'0,0'0] MAX to 146'13431 2024-09-21T10:08:48.168 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:47 smithi032 ceph-mon[101299]: 5.13 starting backfill to osd.3 from (0'0,0'0] MAX to 146'13431 2024-09-21T10:08:48.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:48.212 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:48.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:47 smithi045 ceph-mon[95976]: pgmap v436: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 1173/139650 objects misplaced (0.840%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:08:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:47 smithi045 ceph-mon[95976]: 5.11 scrub starts 2024-09-21T10:08:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:47 smithi045 ceph-mon[95976]: 5.11 scrub ok 2024-09-21T10:08:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:47 smithi045 ceph-mon[95976]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T10:08:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:47 smithi045 ceph-mon[95976]: 5.13 starting backfill to osd.1 from (0'0,0'0] MAX to 146'13431 2024-09-21T10:08:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:47 smithi045 ceph-mon[95976]: 5.13 starting backfill to osd.3 from (0'0,0'0] MAX to 146'13431 2024-09-21T10:08:48.314 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:48.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:48.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:48.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:48.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:48.598 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:48.702 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:48.731 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:48.833 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:48.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.864 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:48.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:48.965 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:48.996 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:48.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:49.059 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:48 smithi032 ceph-mon[105748]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T10:08:49.059 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:08:49.059 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:49.059 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:48 smithi032 ceph-mon[101299]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T10:08:49.059 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:08:49.060 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:49.097 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:49.143 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:49.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:49.245 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:49.280 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:49.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:49.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:48 smithi045 ceph-mon[95976]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T10:08:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:08:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:49.382 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:49.414 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:49.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.425 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:49.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:49.526 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:49.550 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:49.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:49.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:49.676 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:49.693 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:08:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:08:49] "GET /metrics HTTP/1.1" 200 36370 "" "Prometheus/2.51.0" 2024-09-21T10:08:49.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:49.800 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:49.902 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:49.924 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:49.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:50.026 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:50.057 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:50.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:50.123 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:08:50.123 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:08:50.124 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-21T10:08:50.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 110 3.0 2024-09-21T10:08:50.159 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:50.205 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:50.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:49 smithi045 ceph-mon[95976]: pgmap v439: 99 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 96 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2825/137976 objects misplaced (2.047%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:08:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:49 smithi045 ceph-mon[95976]: 5.14 starting backfill to osd.3 from (0'0,0'0] MAX to 149'13537 2024-09-21T10:08:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:49 smithi045 ceph-mon[95976]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T10:08:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:08:50.306 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:50.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.351 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:50.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:50.452 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:49 smithi032 ceph-mon[101299]: pgmap v439: 99 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 96 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2825/137976 objects misplaced (2.047%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:08:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:49 smithi032 ceph-mon[101299]: 5.14 starting backfill to osd.3 from (0'0,0'0] MAX to 149'13537 2024-09-21T10:08:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:49 smithi032 ceph-mon[101299]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T10:08:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:08:50.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:49 smithi032 ceph-mon[105748]: pgmap v439: 99 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 96 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2825/137976 objects misplaced (2.047%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:08:50.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:49 smithi032 ceph-mon[105748]: 5.14 starting backfill to osd.3 from (0'0,0'0] MAX to 149'13537 2024-09-21T10:08:50.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:49 smithi032 ceph-mon[105748]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T10:08:50.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:50.586 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:50.627 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:50.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:50.728 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:50.761 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:50.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:50.857 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:50.857 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:50.863 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:50.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.911 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:50.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:51.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:51.034 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:51.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:51.135 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:51.170 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:50 smithi032 ceph-mon[105748]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T10:08:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:51 smithi032 ceph-mon[105748]: pgmap v442: 99 pgs: 1 peering, 2 remapped+peering, 96 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 15 MiB/s, 245 objects/s recovering 2024-09-21T10:08:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:50 smithi032 ceph-mon[101299]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T10:08:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:51 smithi032 ceph-mon[101299]: pgmap v442: 99 pgs: 1 peering, 2 remapped+peering, 96 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 15 MiB/s, 245 objects/s recovering 2024-09-21T10:08:51.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:50 smithi045 ceph-mon[95976]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T10:08:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:50 smithi045 ceph-mon[95976]: pgmap v442: 99 pgs: 1 peering, 2 remapped+peering, 96 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 15 MiB/s, 245 objects/s recovering 2024-09-21T10:08:51.340 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:51.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:51.441 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:51.478 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:51.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:51.619 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:51.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:51.721 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:51.762 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:51.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:51.863 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:51.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.915 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:51.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:52.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:52.055 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:52.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:52.156 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:52.194 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:52.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:52.297 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:52.337 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:52.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:52.439 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:52.472 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:52.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:52.574 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:52.624 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:52.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:52.726 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:52.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:52.867 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:52.898 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:52.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:53.000 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:53.038 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:53.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:53.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:53.228 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:08:53.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:53.309 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:08:53.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.357 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:53.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:53.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:53 smithi032 ceph-mon[101299]: pgmap v443: 99 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 remapped+peering, 96 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 816/132945 objects misplaced (0.614%); 17 MiB/s, 265 objects/s recovering 2024-09-21T10:08:53.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:53 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2134281726' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:53.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:53 smithi032 ceph-mon[105748]: pgmap v443: 99 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 remapped+peering, 96 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 816/132945 objects misplaced (0.614%); 17 MiB/s, 265 objects/s recovering 2024-09-21T10:08:53.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:53 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2134281726' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:53.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:53.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.506 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:53.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:53 smithi045 ceph-mon[95976]: pgmap v443: 99 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 remapped+peering, 96 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 816/132945 objects misplaced (0.614%); 17 MiB/s, 265 objects/s recovering 2024-09-21T10:08:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:53 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2134281726' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:53.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:53.654 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:53.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:53.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:53.788 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:53.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:53.890 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:53.922 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:53.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:54.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:54.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.069 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:54.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:54.170 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:54.207 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:54.308 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:54.364 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:54.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:54 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:54 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2134281726' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:54 smithi032 ceph-mon[105748]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T10:08:54.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:54 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:54.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:54 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2134281726' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:54.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:54 smithi032 ceph-mon[101299]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T10:08:54.466 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:54.504 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:54 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:54 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2134281726' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:54 smithi045 ceph-mon[95976]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T10:08:54.605 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:54.641 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:54.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:54.743 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:54.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.781 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:54.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:54.882 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:54.911 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:54.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:55.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:55.021 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:55.044 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:55.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:55.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:55.190 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:55.292 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:55.340 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:55.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:55.473 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:55.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:55.575 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:55.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:55.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:55.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:55.751 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:55.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:55.853 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:55.885 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:55.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:55.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:56.033 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:56.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:56.070 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:55 smithi032 ceph-mon[105748]: pgmap v445: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 peering, 96 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 1353/128946 objects misplaced (1.049%); 22 MiB/s, 357 objects/s recovering 2024-09-21T10:08:56.070 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:08:56.070 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:08:56.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:55 smithi032 ceph-mon[101299]: pgmap v445: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 peering, 96 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 1353/128946 objects misplaced (1.049%); 22 MiB/s, 357 objects/s recovering 2024-09-21T10:08:56.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:08:56.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:08:56.135 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:56.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:56.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:55 smithi045 ceph-mon[95976]: pgmap v445: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 peering, 96 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 1353/128946 objects misplaced (1.049%); 22 MiB/s, 357 objects/s recovering 2024-09-21T10:08:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:08:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:08:56.302 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:56.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:56.404 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:56.437 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:56.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:56.448 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:08:56.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:56.571 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:56.673 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:56.695 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:56.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:56.797 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:56.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.830 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:56.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:56.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:56.955 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:57.057 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:57.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:57.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:57.193 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:56 smithi032 ceph-mon[101299]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T10:08:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2062872388' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:56 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:08:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:56 smithi032 ceph-mon[105748]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T10:08:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2062872388' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:56 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:08:57.226 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:57.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:56 smithi045 ceph-mon[95976]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T10:08:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2062872388' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:56 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:08:57.328 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:57.350 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:57.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:57.451 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:57.474 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:57.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:57.575 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:57.597 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:57.699 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:57.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:57.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:57.823 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:57.844 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:57.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:57.946 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:57.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:58.070 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:58.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:58.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:58.155 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:58.156 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:58.157 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:58.157 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:58.157 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:58.157 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:58.202 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:58.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:57 smithi032 ceph-mon[101299]: pgmap v447: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 1353/121932 objects misplaced (1.110%); 17 MiB/s, 270 objects/s recovering 2024-09-21T10:08:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:57 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:08:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:57 smithi032 ceph-mon[101299]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T10:08:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:57 smithi032 ceph-mon[105748]: pgmap v447: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 1353/121932 objects misplaced (1.110%); 17 MiB/s, 270 objects/s recovering 2024-09-21T10:08:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:57 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:08:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:57 smithi032 ceph-mon[105748]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T10:08:58.230 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:57 smithi045 ceph-mon[95976]: pgmap v447: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 1353/121932 objects misplaced (1.110%); 17 MiB/s, 270 objects/s recovering 2024-09-21T10:08:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:57 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:08:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:57 smithi045 ceph-mon[95976]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T10:08:58.332 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:58.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.355 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:58.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:58.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:58.479 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:58.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:58.581 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:58.603 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:58.704 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:58.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:58.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:58.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:58.851 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:58.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:58.953 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:58.975 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:58.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:59.077 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:59.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.101 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:59.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:59.144 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:59.184 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:58 smithi032 ceph-mon[101299]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T10:08:59.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:58 smithi032 ceph-mon[101299]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T10:08:59.185 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:58 smithi032 ceph-mon[105748]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T10:08:59.185 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:58 smithi032 ceph-mon[105748]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T10:08:59.201 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:59.224 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:59.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:58 smithi045 ceph-mon[95976]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T10:08:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:58 smithi045 ceph-mon[95976]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T10:08:59.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:59.348 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:59.450 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:59.473 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:59.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:59.575 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:59.616 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:59.718 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:59.751 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:08:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:59.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:59.886 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:08:59.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:59.940 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:08:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:08:59] "GET /metrics HTTP/1.1" 200 36370 "" "Prometheus/2.51.0" 2024-09-21T10:08:59.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:00.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.023 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:00.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:00.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:00.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:00.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:59 smithi032 ceph-mon[105748]: pgmap v450: 99 pgs: 1 peering, 1 active+remapped+backfilling, 97 active+clean; 2.4 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 416/119268 objects misplaced (0.349%); 18 MiB/s, 289 objects/s recovering 2024-09-21T10:09:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:08:59 smithi032 ceph-mon[105748]: 5.15 starting backfill to osd.5 from (0'0,0'0] MAX to 155'13603 2024-09-21T10:09:00.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:59 smithi032 ceph-mon[101299]: pgmap v450: 99 pgs: 1 peering, 1 active+remapped+backfilling, 97 active+clean; 2.4 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 416/119268 objects misplaced (0.349%); 18 MiB/s, 289 objects/s recovering 2024-09-21T10:09:00.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:08:59 smithi032 ceph-mon[101299]: 5.15 starting backfill to osd.5 from (0'0,0'0] MAX to 155'13603 2024-09-21T10:09:00.258 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:59 smithi045 ceph-mon[95976]: pgmap v450: 99 pgs: 1 peering, 1 active+remapped+backfilling, 97 active+clean; 2.4 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 416/119268 objects misplaced (0.349%); 18 MiB/s, 289 objects/s recovering 2024-09-21T10:09:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:08:59 smithi045 ceph-mon[95976]: 5.15 starting backfill to osd.5 from (0'0,0'0] MAX to 155'13603 2024-09-21T10:09:00.293 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:00.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:00.394 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:00.428 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:00.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:00.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:00.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:00.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:00.700 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:00.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:00.801 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:00.829 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:00.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:00.840 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:09:00.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:00.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.971 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:00.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:01.072 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:01.116 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:01.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:01.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:01.255 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:01.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:01.357 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:01.394 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:01.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:01.495 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:01.520 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:01.622 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:01.667 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:01.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:01.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.799 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:01.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:01.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:01.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.944 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:01.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:01 smithi032 ceph-mon[105748]: pgmap v452: 99 pgs: 1 peering, 1 active+remapped+backfilling, 97 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 416/111372 objects misplaced (0.374%); 9.8 MiB/s, 156 objects/s recovering 2024-09-21T10:09:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:01 smithi032 ceph-mon[101299]: pgmap v452: 99 pgs: 1 peering, 1 active+remapped+backfilling, 97 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 416/111372 objects misplaced (0.374%); 9.8 MiB/s, 156 objects/s recovering 2024-09-21T10:09:02.046 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:02.083 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:02.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:02.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:02.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:01 smithi045 ceph-mon[95976]: pgmap v452: 99 pgs: 1 peering, 1 active+remapped+backfilling, 97 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 416/111372 objects misplaced (0.374%); 9.8 MiB/s, 156 objects/s recovering 2024-09-21T10:09:02.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:02.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.357 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:02.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:02.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:02.488 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:02.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:02.589 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:02.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:02.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:02.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:02.762 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:02.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:02.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:02.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:02.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:03.004 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:03.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:03.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:03.182 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:03.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:03.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:02 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:09:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:02 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:09:03.284 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:02 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:09:03.323 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:03.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:03.425 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:03.452 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:03.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:03.554 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:03.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:03.683 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:03.706 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:03.807 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:03.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:03.933 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:03.958 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:03.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:04.060 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:04.084 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:04.173 INFO:teuthology.orchestra.run.smithi032.stdout:moved 26 / 297 (8.75421%) 2024-09-21T10:09:04.173 INFO:teuthology.orchestra.run.smithi032.stdout:avg 37.125 2024-09-21T10:09:04.173 INFO:teuthology.orchestra.run.smithi032.stdout:stddev 5.10973 -> 5.94637 (expected baseline 5.69951) 2024-09-21T10:09:04.173 INFO:teuthology.orchestra.run.smithi032.stdout:min osd.2 with 27 -> 32 pgs (0.727273 -> 0.861953 * mean) 2024-09-21T10:09:04.173 INFO:teuthology.orchestra.run.smithi032.stdout:max osd.4 with 44 -> 38 pgs (1.18519 -> 1.02357 * mean) 2024-09-21T10:09:04.173 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:09:04.174 INFO:teuthology.orchestra.run.smithi032.stdout:oload 110 2024-09-21T10:09:04.174 INFO:teuthology.orchestra.run.smithi032.stdout:max_change 3 2024-09-21T10:09:04.174 INFO:teuthology.orchestra.run.smithi032.stdout:max_change_osds 4 2024-09-21T10:09:04.174 INFO:teuthology.orchestra.run.smithi032.stdout:average_utilization 0.0113 2024-09-21T10:09:04.174 INFO:teuthology.orchestra.run.smithi032.stdout:overload_utilization 0.0125 2024-09-21T10:09:04.174 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 weight 1.0000 -> 0.7057 2024-09-21T10:09:04.174 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 weight 1.0000 -> 0.7882 2024-09-21T10:09:04.174 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 weight 1.0000 -> 0.8144 2024-09-21T10:09:04.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:04.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:03 smithi045 ceph-mon[95976]: pgmap v453: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 892/109548 objects misplaced (0.814%); 11 MiB/s, 184 objects/s recovering 2024-09-21T10:09:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:09:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:03 smithi045 ceph-mon[95976]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T10:09:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'53373','4':'51657','5':'46252'}"} : dispatch 2024-09-21T10:09:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:04.311 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:04.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:04.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:04.435 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:04.456 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[101299]: pgmap v453: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 892/109548 objects misplaced (0.814%); 11 MiB/s, 184 objects/s recovering 2024-09-21T10:09:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:09:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[101299]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T10:09:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'53373','4':'51657','5':'46252'}"} : dispatch 2024-09-21T10:09:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[105748]: pgmap v453: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 892/109548 objects misplaced (0.814%); 11 MiB/s, 184 objects/s recovering 2024-09-21T10:09:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:09:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[105748]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T10:09:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'53373','4':'51657','5':'46252'}"} : dispatch 2024-09-21T10:09:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:04.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:04.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:04.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:04.582 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:04.684 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:04.706 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:04.710 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:09:04.711 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:09:04.711 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:09:04.711 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:09:04.711 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:09:04.808 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:04.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.832 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:04.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:04.902 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:04.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:04.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:04.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:04.978 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:04 smithi032 ceph-mon[101299]: from='client.34839 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:04.978 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'53373','4':'51657','5':'46252'}"}]': finished 2024-09-21T10:09:04.978 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:04 smithi032 ceph-mon[101299]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T10:09:04.978 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-21T10:09:04.982 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:04 smithi032 ceph-mon[105748]: from='client.34839 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:04.982 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'53373','4':'51657','5':'46252'}"}]': finished 2024-09-21T10:09:04.983 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:04 smithi032 ceph-mon[105748]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T10:09:04.983 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-21T10:09:05.057 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:05.081 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:05.183 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:05.207 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:05.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:04 smithi045 ceph-mon[95976]: from='client.34839 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'53373','4':'51657','5':'46252'}"}]': finished 2024-09-21T10:09:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:04 smithi045 ceph-mon[95976]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T10:09:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-21T10:09:05.308 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:05.332 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:05.434 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:05.459 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:05.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:09:05.561 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:05.597 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:05.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:05.699 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:05.722 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:05.823 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:05.848 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:05.950 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:05.978 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:05.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:06.080 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:06.083 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-21T10:09:06.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 110 1.0 2024-09-21T10:09:06.104 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:06.112 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[101299]: pgmap v456: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 892/105873 objects misplaced (0.843%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:09:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-21T10:09:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[101299]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T10:09:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[101299]: 5.16 starting backfill to osd.1 from (0'0,0'0] MAX to 158'13724 2024-09-21T10:09:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[101299]: 5.16 starting backfill to osd.3 from (0'0,0'0] MAX to 158'13724 2024-09-21T10:09:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[101299]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T10:09:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[105748]: pgmap v456: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 892/105873 objects misplaced (0.843%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:09:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-21T10:09:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[105748]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T10:09:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[105748]: 5.16 starting backfill to osd.1 from (0'0,0'0] MAX to 158'13724 2024-09-21T10:09:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[105748]: 5.16 starting backfill to osd.3 from (0'0,0'0] MAX to 158'13724 2024-09-21T10:09:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:05 smithi032 ceph-mon[105748]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T10:09:06.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:06.265 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:06.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:05 smithi045 ceph-mon[95976]: pgmap v456: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 892/105873 objects misplaced (0.843%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:09:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:05 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-21T10:09:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:05 smithi045 ceph-mon[95976]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T10:09:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:05 smithi045 ceph-mon[95976]: 5.16 starting backfill to osd.1 from (0'0,0'0] MAX to 158'13724 2024-09-21T10:09:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:05 smithi045 ceph-mon[95976]: 5.16 starting backfill to osd.3 from (0'0,0'0] MAX to 158'13724 2024-09-21T10:09:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:05 smithi045 ceph-mon[95976]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T10:09:06.369 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:06.398 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:06.500 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:06.539 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:06.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:06.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:06.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:06.781 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:06.828 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:06.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:06.929 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:06.974 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:07.076 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:07.114 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:07.149 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:07.150 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: 5.11 starting backfill to osd.0 from (0'0,0'0] MAX to 159'13557 2024-09-21T10:09:07.150 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 159'13818 2024-09-21T10:09:07.150 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: 5.b starting backfill to osd.7 from (0'0,0'0] MAX to 159'14633 2024-09-21T10:09:07.150 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 159'14646 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: 5.5 starting backfill to osd.6 from (0'0,0'0] MAX to 159'13797 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: 5.7 starting backfill to osd.4 from (0'0,0'0] MAX to 159'14033 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: 5.17 starting backfill to osd.3 from (0'0,0'0] MAX to 159'13983 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: 5.5 starting backfill to osd.7 from (0'0,0'0] MAX to 159'13797 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: 5.17 starting backfill to osd.7 from (0'0,0'0] MAX to 159'13983 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[105748]: pgmap v459: 99 pgs: 1 remapped+peering, 4 unknown, 12 peering, 1 active+remapped+backfilling, 81 active+clean; 2.0 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 892/98544 objects misplaced (0.905%) 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: 5.11 starting backfill to osd.0 from (0'0,0'0] MAX to 159'13557 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 159'13818 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: 5.b starting backfill to osd.7 from (0'0,0'0] MAX to 159'14633 2024-09-21T10:09:07.151 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 159'14646 2024-09-21T10:09:07.152 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: 5.5 starting backfill to osd.6 from (0'0,0'0] MAX to 159'13797 2024-09-21T10:09:07.152 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: 5.7 starting backfill to osd.4 from (0'0,0'0] MAX to 159'14033 2024-09-21T10:09:07.152 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: 5.17 starting backfill to osd.3 from (0'0,0'0] MAX to 159'13983 2024-09-21T10:09:07.152 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: 5.5 starting backfill to osd.7 from (0'0,0'0] MAX to 159'13797 2024-09-21T10:09:07.152 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: 5.17 starting backfill to osd.7 from (0'0,0'0] MAX to 159'13983 2024-09-21T10:09:07.152 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:06 smithi032 ceph-mon[101299]: pgmap v459: 99 pgs: 1 remapped+peering, 4 unknown, 12 peering, 1 active+remapped+backfilling, 81 active+clean; 2.0 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 892/98544 objects misplaced (0.905%) 2024-09-21T10:09:07.215 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:07.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:07.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: 5.11 starting backfill to osd.0 from (0'0,0'0] MAX to 159'13557 2024-09-21T10:09:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 159'13818 2024-09-21T10:09:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: 5.b starting backfill to osd.7 from (0'0,0'0] MAX to 159'14633 2024-09-21T10:09:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 159'14646 2024-09-21T10:09:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: 5.5 starting backfill to osd.6 from (0'0,0'0] MAX to 159'13797 2024-09-21T10:09:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: 5.7 starting backfill to osd.4 from (0'0,0'0] MAX to 159'14033 2024-09-21T10:09:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: 5.17 starting backfill to osd.3 from (0'0,0'0] MAX to 159'13983 2024-09-21T10:09:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: 5.5 starting backfill to osd.7 from (0'0,0'0] MAX to 159'13797 2024-09-21T10:09:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: 5.17 starting backfill to osd.7 from (0'0,0'0] MAX to 159'13983 2024-09-21T10:09:07.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:06 smithi045 ceph-mon[95976]: pgmap v459: 99 pgs: 1 remapped+peering, 4 unknown, 12 peering, 1 active+remapped+backfilling, 81 active+clean; 2.0 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 892/98544 objects misplaced (0.905%) 2024-09-21T10:09:07.361 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:07.405 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:07.507 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:07.548 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:07.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:07.650 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:07.691 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:07.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:07.793 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:07.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.833 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:07.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:07.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:07.971 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:08.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:08.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:08.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:08.211 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:08.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:07 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-21T10:09:08.233 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:07 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-21T10:09:08.255 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:08.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:07 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-21T10:09:08.357 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:08.393 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:08.498 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:08.535 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:08.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:08.580 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:08.637 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:08.672 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:08.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:08.774 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:08.822 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:08.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:08.924 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:08.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:09.063 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:09.101 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:09.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:09.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:09.252 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:09.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:09 smithi045 ceph-mon[95976]: pgmap v460: 99 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 4 unknown, 12 peering, 1 active+remapped+backfilling, 80 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 1896/97620 objects misplaced (1.942%) 2024-09-21T10:09:09.308 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:09 smithi032 ceph-mon[105748]: pgmap v460: 99 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 4 unknown, 12 peering, 1 active+remapped+backfilling, 80 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 1896/97620 objects misplaced (1.942%) 2024-09-21T10:09:09.308 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:09 smithi032 ceph-mon[101299]: pgmap v460: 99 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 4 unknown, 12 peering, 1 active+remapped+backfilling, 80 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 1896/97620 objects misplaced (1.942%) 2024-09-21T10:09:09.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:09.430 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:09.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:09.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:09.578 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:09.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:09.623 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:09:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:09:09] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.51.0" 2024-09-21T10:09:09.680 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:09.716 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:09.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:09.817 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:09.858 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:09.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:10.007 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:10.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:10.109 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:10.152 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:10.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:10.254 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:10.297 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:10.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:10.398 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:10.423 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:10.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:10.525 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:10.563 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:10.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:10.665 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:10.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.712 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:10.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:10.812 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:10.843 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:10.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:10.944 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:10.979 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:10.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:11.081 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:11.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:11.212 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:11.252 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:11.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:11 smithi045 ceph-mon[95976]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T10:09:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:11 smithi045 ceph-mon[95976]: pgmap v462: 99 pgs: 4 active+remapped+backfill_wait, 4 active+remapped+backfilling, 91 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 9137/89364 objects misplaced (10.224%); 30 MiB/s, 480 objects/s recovering 2024-09-21T10:09:11.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:11.395 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:11.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:11 smithi032 ceph-mon[105748]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T10:09:11.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:11 smithi032 ceph-mon[105748]: pgmap v462: 99 pgs: 4 active+remapped+backfill_wait, 4 active+remapped+backfilling, 91 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 9137/89364 objects misplaced (10.224%); 30 MiB/s, 480 objects/s recovering 2024-09-21T10:09:11.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:11 smithi032 ceph-mon[101299]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T10:09:11.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:11 smithi032 ceph-mon[101299]: pgmap v462: 99 pgs: 4 active+remapped+backfill_wait, 4 active+remapped+backfilling, 91 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 9137/89364 objects misplaced (10.224%); 30 MiB/s, 480 objects/s recovering 2024-09-21T10:09:11.496 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:11.535 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:11.636 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:11.673 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:11.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:11.775 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:11.809 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:11.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:11.910 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:11.938 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:11.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:12.040 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:12.074 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:12.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:12.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:12.200 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:12.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:12 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2024-09-21T10:09:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:12 smithi032 ceph-mon[105748]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T10:09:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:12 smithi032 ceph-mon[105748]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T10:09:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:12 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2024-09-21T10:09:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:12 smithi032 ceph-mon[101299]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T10:09:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:12 smithi032 ceph-mon[101299]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T10:09:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:12 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2024-09-21T10:09:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:12 smithi045 ceph-mon[95976]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T10:09:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:12 smithi045 ceph-mon[95976]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T10:09:12.302 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:12.331 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:12.433 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:12.456 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:12.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:12.558 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:12.597 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:12.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:12.699 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:12.729 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:12.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:12.830 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:12.855 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:12.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:12.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:12.982 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:12.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:13.083 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:13.109 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:13.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:13.211 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:13.236 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:13.338 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:13.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:13.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:13.462 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:13.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:13.589 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:13.614 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:13.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:13.716 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:13.751 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:13 smithi045 ceph-mon[95976]: pgmap v465: 99 pgs: 3 active+remapped+backfill_wait, 5 active+remapped+backfilling, 91 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 8687/87624 objects misplaced (9.914%); 33 MiB/s, 529 objects/s recovering 2024-09-21T10:09:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:13 smithi045 ceph-mon[95976]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T10:09:13.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:13.886 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:13.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:13 smithi032 ceph-mon[101299]: pgmap v465: 99 pgs: 3 active+remapped+backfill_wait, 5 active+remapped+backfilling, 91 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 8687/87624 objects misplaced (9.914%); 33 MiB/s, 529 objects/s recovering 2024-09-21T10:09:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:13 smithi032 ceph-mon[101299]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T10:09:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:13 smithi032 ceph-mon[105748]: pgmap v465: 99 pgs: 3 active+remapped+backfill_wait, 5 active+remapped+backfilling, 91 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 8687/87624 objects misplaced (9.914%); 33 MiB/s, 529 objects/s recovering 2024-09-21T10:09:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:13 smithi032 ceph-mon[105748]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T10:09:13.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:14.029 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:14.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:14.166 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:14.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:14.268 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:14.308 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:14.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:14.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:14.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:14.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:14.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:14.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:14.684 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:14.707 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:14 smithi045 ceph-mon[95976]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T10:09:14.725 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:14.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:14.827 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:14.849 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:14.951 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:14.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:14 smithi032 ceph-mon[101299]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T10:09:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:14 smithi032 ceph-mon[105748]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T10:09:14.985 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:14.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:15.087 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:15.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:15.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:15.230 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:15.270 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:15.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:15.372 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:15.378 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:09:15.378 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213387 5.10 5:0b016a26:::benchmark_data_smithi032_140417_object79996:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881220+0000", 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489941469999998, 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1090799090000001, 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.379 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213387 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881220+0000", 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881220+0000", 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.380 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881222+0000", 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5999999999999999e-06 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881226+0000", 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8509999999999998e-06 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881240+0000", 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3181e-05 2024-09-21T10:09:15.381 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881250+0000", 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0917000000000001e-05 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881400+0000", 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000150037 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881407+0000", 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1399999999999997e-06 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.382 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.985315+0000", 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1039081820000001 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.985448+0000", 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000133683 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.985734+0000", 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000285848 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.383 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.989937+0000", 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0042021990000000002 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.989960+0000", 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3519999999999998e-05 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.989991+0000", 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0861000000000001e-05 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.384 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990063+0000", 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1752e-05 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990185+0000", 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000122099 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990230+0000", 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5274000000000001e-05 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.385 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990300+0000", 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9764999999999994e-05 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213398 5.10 5:0fddfabd:::benchmark_data_smithi032_140417_object80007:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881293+0000", 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489868094000002, 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1094213850000001, 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.386 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213398 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881293+0000", 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881293+0000", 2024-09-21T10:09:15.387 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881294+0000", 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7400000000000003e-07 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881297+0000", 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0429999999999999e-06 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881307+0000", 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6260000000000007e-06 2024-09-21T10:09:15.388 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881314+0000", 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2239999999999998e-06 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881432+0000", 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011845099999999999 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.389 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881440+0000", 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1699999999999997e-06 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.987436+0000", 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1059961190000001 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.390 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.987545+0000", 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010817799999999999 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.987720+0000", 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017514599999999999 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990054+0000", 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0023337969999999999 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.391 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990099+0000", 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5080000000000002e-05 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990149+0000", 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0105000000000002e-05 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990157+0000", 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4209999999999995e-06 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.392 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990574+0000", 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000416532 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990615+0000", 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1718999999999998e-05 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990715+0000", 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.9199999999999999e-05 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.393 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213409 5.10 5:0987abc7:::benchmark_data_smithi032_140417_object80018:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881331+0000", 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489830363999999, 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1095653409999999, 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.394 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213409 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881331+0000", 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881331+0000", 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.395 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881331+0000", 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5999999999999999e-07 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881343+0000", 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.149e-05 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881351+0000", 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5019999999999992e-06 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.396 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881357+0000", 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5130000000000004e-06 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881470+0000", 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000113038 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881474+0000", 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5269999999999999e-06 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.397 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.988748+0000", 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1072741420000001 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.988851+0000", 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010298699999999999 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.989049+0000", 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00019869199999999999 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.398 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990161+0000", 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001111976 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990497+0000", 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00033608100000000001 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990528+0000", 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0617999999999997e-05 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.399 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990751+0000", 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000222461 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990786+0000", 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5185000000000003e-05 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990818+0000", 2024-09-21T10:09:15.400 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1897999999999999e-05 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990896+0000", 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8770999999999995e-05 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.401 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213416 5.10 5:0a096f08:::benchmark_data_smithi032_140417_object80025:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881383+0000", 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489778428999998, 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1212008090000001, 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213416 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.402 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881383+0000", 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881383+0000", 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881384+0000", 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5700000000000002e-07 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.403 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881386+0000", 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6070000000000002e-06 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881394+0000", 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2360000000000004e-06 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881402+0000", 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.435e-06 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.404 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.405 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881504+0000", 2024-09-21T10:09:15.405 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000102328 2024-09-21T10:09:15.405 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.405 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.405 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.405 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881512+0000", 2024-09-21T10:09:15.405 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4000000000000003e-06 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.990977+0000", 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.10946586 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.991058+0000", 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0634000000000001e-05 2024-09-21T10:09:15.406 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.991280+0000", 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000222012 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996365+0000", 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.005085099 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996418+0000", 2024-09-21T10:09:15.407 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3044000000000002e-05 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996455+0000", 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6696999999999998e-05 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996626+0000", 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000171516 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.408 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002387+0000", 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0057610819999999998 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002447+0000", 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9939999999999999e-05 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002584+0000", 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013636200000000001 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.409 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213453 5.10 5:0b0a6832:::benchmark_data_smithi032_140417_object80062:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881413+0000", 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489748235999997, 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1215317650000001, 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213453 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.410 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881413+0000", 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881413+0000", 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881414+0000", 2024-09-21T10:09:15.411 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9500000000000003e-07 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881416+0000", 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4279999999999999e-06 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881423+0000", 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6780000000000003e-06 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881428+0000", 2024-09-21T10:09:15.412 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5629999999999999e-06 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881537+0000", 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000108986 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881541+0000", 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2289999999999999e-06 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.413 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.993034+0000", 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1114936929999999 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.993162+0000", 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012785800000000001 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.993549+0000", 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000386733 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.414 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996501+0000", 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0029516970000000001 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996545+0000", 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4397000000000002e-05 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996586+0000", 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0710000000000002e-05 2024-09-21T10:09:15.415 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996674+0000", 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8539000000000001e-05 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002741+0000", 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.006066648 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002805+0000", 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4061999999999998e-05 2024-09-21T10:09:15.416 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002945+0000", 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000140049 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213458 5.10 5:0a624e55:::benchmark_data_smithi032_140417_object80067:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881436+0000", 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489725460999999, 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.129175686, 2024-09-21T10:09:15.417 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213458 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881436+0000", 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.418 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881436+0000", 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881436+0000", 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1100000000000001e-07 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881438+0000", 2024-09-21T10:09:15.419 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0279999999999999e-06 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881445+0000", 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2809999999999997e-06 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881450+0000", 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1699999999999996e-06 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881572+0000", 2024-09-21T10:09:15.420 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012207400000000001 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881579+0000", 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2080000000000003e-06 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996709+0000", 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.115129622 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.996834+0000", 2024-09-21T10:09:15.421 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012510000000000001 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:34.997188+0000", 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00035390399999999999 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002649+0000", 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0054615419999999998 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.422 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002720+0000", 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0862e-05 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002958+0000", 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00023812200000000001 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.003267+0000", 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00030874400000000001 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.423 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.010434+0000", 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0071671030000000002 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.010491+0000", 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7178e-05 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.010612+0000", 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000120337 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.424 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213507 5.10 5:0f0135df:::benchmark_data_smithi032_140417_object80116:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881459+0000", 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489702532000003, 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1310787200000001, 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213507 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.425 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881459+0000", 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881459+0000", 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881459+0000", 2024-09-21T10:09:15.426 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9800000000000004e-07 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881461+0000", 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.764e-06 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881465+0000", 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8770000000000003e-06 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881468+0000", 2024-09-21T10:09:15.427 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.898e-06 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881604+0000", 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013575800000000001 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881607+0000", 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2720000000000002e-06 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.000649+0000", 2024-09-21T10:09:15.428 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.119041996 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.000792+0000", 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014281 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.001072+0000", 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000280318 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.002965+0000", 2024-09-21T10:09:15.429 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001892555 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.005377+0000", 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002411977 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.005448+0000", 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1554e-05 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.005522+0000", 2024-09-21T10:09:15.430 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4270999999999994e-05 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012309+0000", 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0067862189999999996 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012381+0000", 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2212999999999995e-05 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.431 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012538+0000", 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015673999999999999 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213582 5.10 5:09a31ad5:::benchmark_data_smithi032_140417_object80191:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881472+0000", 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489689314000003, 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.13145165, 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.432 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213582 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881472+0000", 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881472+0000", 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.433 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881472+0000", 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.67e-07 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881473+0000", 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0869999999999999e-06 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881477+0000", 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5839999999999999e-06 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.434 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881482+0000", 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7620000000000003e-06 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881637+0000", 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015499399999999999 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881644+0000", 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2649999999999999e-06 2024-09-21T10:09:15.435 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.003307+0000", 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.121662715 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.003423+0000", 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000116385 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.003671+0000", 2024-09-21T10:09:15.436 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000247507 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.007582+0000", 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0039110330000000004 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.009630+0000", 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0020479629999999999 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.009681+0000", 2024-09-21T10:09:15.437 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1251000000000001e-05 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012638+0000", 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0029571200000000001 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012719+0000", 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1437e-05 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.438 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012783+0000", 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3552000000000004e-05 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012924+0000", 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014072800000000001 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213626 5.10 5:0d918b32:::benchmark_data_smithi032_140417_object80235:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881498+0000", 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489663483999998, 2024-09-21T10:09:15.439 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.136965088, 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213626 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881498+0000", 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.440 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881498+0000", 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881498+0000", 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.27e-07 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881500+0000", 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0320000000000002e-06 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881507+0000", 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4829999999999996e-06 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881512+0000", 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3709999999999999e-06 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881672+0000", 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000159581 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.442 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881675+0000", 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2739999999999999e-06 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.005573+0000", 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1238976119999999 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.005774+0000", 2024-09-21T10:09:15.443 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00020122599999999999 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.006077+0000", 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00030284499999999999 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.009729+0000", 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0036518359999999999 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012586+0000", 2024-09-21T10:09:15.444 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0028574270000000001 2024-09-21T10:09:15.445 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.445 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.445 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.445 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012679+0000", 2024-09-21T10:09:15.445 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2761000000000002e-05 2024-09-21T10:09:15.445 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.445 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.445 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.446 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012959+0000", 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00028057499999999998 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018240+0000", 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0052808869999999997 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018345+0000", 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000104393 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.447 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018463+0000", 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011835799999999999 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213633 5.10 5:0af63318:::benchmark_data_smithi032_140417_object80242:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881537+0000", 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489624192000001, 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1371577020000001, 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.448 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213633 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881537+0000", 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881537+0000", 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.449 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881538+0000", 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9999999999999998e-07 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881540+0000", 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.074e-06 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881546+0000", 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2559999999999999e-06 2024-09-21T10:09:15.450 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881552+0000", 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.677e-06 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881712+0000", 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015995899999999999 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881725+0000", 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3366e-05 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.451 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.007633+0000", 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.125908476 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.007781+0000", 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000147324 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.008067+0000", 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00028656800000000001 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.452 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.012969+0000", 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0049018050000000004 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.013037+0000", 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7490999999999995e-05 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.013099+0000", 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2378000000000005e-05 2024-09-21T10:09:15.453 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.013174+0000", 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5195000000000001e-05 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018543+0000", 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0053690070000000003 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018596+0000", 2024-09-21T10:09:15.454 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2661999999999998e-05 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018695+0000", 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.9063999999999999e-05 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213651 5.10 5:0dc8e90d:::benchmark_data_smithi032_140417_object80260:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881565+0000", 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489596036000002, 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1442286720000001, 2024-09-21T10:09:15.455 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213651 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881565+0000", 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.456 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881565+0000", 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881566+0000", 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5400000000000002e-07 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881568+0000", 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.99e-06 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.457 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881574+0000", 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6059999999999999e-06 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881580+0000", 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.361e-06 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.458 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881752+0000", 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017201799999999999 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881755+0000", 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.23e-06 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.013225+0000", 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.131469727 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.459 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.013362+0000", 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013680699999999999 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.013722+0000", 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00036017800000000001 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018159+0000", 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0044375270000000001 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.460 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018167+0000", 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9619999999999993e-06 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025356+0000", 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0071883379999999998 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025442+0000", 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6712000000000006e-05 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.461 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025599+0000", 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015637800000000001 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025685+0000", 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6600000000000004e-05 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025794+0000", 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010878399999999999 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.462 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213671 5.10 5:0dd1faee:::benchmark_data_smithi032_140417_object80280:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881603+0000", 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489557998999999, 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.144659519, 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.463 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213671 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881603+0000", 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881603+0000", 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881604+0000", 2024-09-21T10:09:15.464 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0400000000000002e-07 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881609+0000", 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5289999999999999e-06 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881617+0000", 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.807e-06 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881623+0000", 2024-09-21T10:09:15.465 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.699e-06 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881787+0000", 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016437900000000001 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881794+0000", 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2579999999999998e-06 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.466 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.015548+0000", 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1337532379999999 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.015668+0000", 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012012599999999999 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.015989+0000", 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00032120100000000002 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.467 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018760+0000", 2024-09-21T10:09:15.468 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002770631 2024-09-21T10:09:15.468 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.468 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.468 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.469 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018776+0000", 2024-09-21T10:09:15.469 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6458999999999999e-05 2024-09-21T10:09:15.469 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.469 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.469 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025526+0000", 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0067500980000000004 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025984+0000", 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00045745900000000001 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026058+0000", 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4740999999999995e-05 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.470 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026112+0000", 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3661e-05 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026263+0000", 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000150829 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213799 5.10 5:0fc41b04:::benchmark_data_smithi032_140417_object80408:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.471 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881631+0000", 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489530133000002, 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.144984164, 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213799 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881631+0000", 2024-09-21T10:09:15.472 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881631+0000", 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881632+0000", 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7999999999999996e-07 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881634+0000", 2024-09-21T10:09:15.473 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1059999999999998e-06 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881640+0000", 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1809999999999998e-06 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881645+0000", 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2689999999999999e-06 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.474 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881820+0000", 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017484 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881823+0000", 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.128e-06 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018820+0000", 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1369964160000001 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.475 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.018946+0000", 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000126375 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.019301+0000", 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00035488300000000001 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025542+0000", 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0062410879999999997 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.476 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025871+0000", 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00032861499999999998 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025932+0000", 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0875999999999998e-05 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026021+0000", 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9948e-05 2024-09-21T10:09:15.477 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026413+0000", 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000391429 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026485+0000", 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2441999999999995e-05 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026615+0000", 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013008799999999999 2024-09-21T10:09:15.478 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213835 5.10 5:0f3d838d:::benchmark_data_smithi032_140417_object80444:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881669+0000", 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489492267999999, 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1452891549999999, 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.479 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213835 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881669+0000", 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881669+0000", 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881670+0000", 2024-09-21T10:09:15.480 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0999999999999999e-07 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881672+0000", 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9870000000000002e-06 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881676+0000", 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7060000000000003e-06 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881681+0000", 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2259999999999996e-06 2024-09-21T10:09:15.481 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881849+0000", 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016801500000000001 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881859+0000", 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6530000000000006e-06 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.023292+0000", 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.141433256 2024-09-21T10:09:15.482 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.023411+0000", 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000118208 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.023763+0000", 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00035250499999999997 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.025884+0000", 2024-09-21T10:09:15.483 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0021207380000000001 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026657+0000", 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00077301099999999999 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026708+0000", 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1248999999999997e-05 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026745+0000", 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6742000000000002e-05 2024-09-21T10:09:15.484 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026781+0000", 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6109999999999998e-05 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026841+0000", 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0510000000000002e-05 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026958+0000", 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000116829 2024-09-21T10:09:15.485 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213850 5.10 5:09c38d9d:::benchmark_data_smithi032_140417_object80459:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881703+0000", 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489458386000003, 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.163920249, 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.486 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213850 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881703+0000", 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881703+0000", 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.487 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881703+0000", 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.58e-07 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881706+0000", 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0829999999999998e-06 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881713+0000", 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0029999999999997e-06 2024-09-21T10:09:15.488 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881722+0000", 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2550000000000003e-06 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881942+0000", 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00021974599999999999 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881951+0000", 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7459999999999995e-06 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.489 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.026985+0000", 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.145033599 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.027092+0000", 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000106974 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.490 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.027411+0000", 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000319217 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.043340+0000", 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.015928761999999999 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.043389+0000", 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9663999999999997e-05 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.491 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.043638+0000", 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024848600000000002 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.043719+0000", 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1021999999999998e-05 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.045439+0000", 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001720051 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.492 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.045500+0000", 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1005999999999998e-05 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.045623+0000", 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000123177 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:213854 5.10 5:0863b2d6:::benchmark_data_smithi032_140417_object80463:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e140)", 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:33.881732+0000", 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 41.489429141000002, 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.164119763, 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 213854 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.494 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881732+0000", 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881732+0000", 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881733+0000", 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4299999999999998e-07 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.495 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881735+0000", 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2000000000000001e-06 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881741+0000", 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4670000000000001e-06 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881745+0000", 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1489999999999996e-06 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.496 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881981+0000", 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00023500600000000001 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:33.881988+0000", 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8730000000000007e-06 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.029392+0000", 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.147403843 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.497 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.029514+0000", 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000121797 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4,6,7", 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.029837+0000", 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00032301200000000002 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.043441+0000", 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013603829 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.498 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.043601+0000", 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016060299999999999 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.043683+0000", 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1834000000000003e-05 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.043754+0000", 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0943000000000002e-05 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.499 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.500 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.500 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.045664+0000", 2024-09-21T10:09:15.500 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001909617 2024-09-21T10:09:15.500 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.500 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.500 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:15.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:15.501 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.501 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.045716+0000", 2024-09-21T10:09:15.501 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1694999999999999e-05 2024-09-21T10:09:15.501 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.501 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.501 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.501 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.045852+0000", 2024-09-21T10:09:15.501 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013645199999999999 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:234362 5.17 5:ecae6b7c:::benchmark_data_smithi032_140417_object100971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:05.087124+0000", 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 10.284037415, 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0982806000000001, 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.502 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 234362 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.087124+0000", 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.087124+0000", 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.503 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.087126+0000", 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7850000000000001e-06 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.087131+0000", 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5799999999999999e-06 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.087144+0000", 2024-09-21T10:09:15.504 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2292e-05 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.087149+0000", 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7049999999999996e-06 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.087207+0000", 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7618999999999998e-05 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.087215+0000", 2024-09-21T10:09:15.505 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5189999999999992e-06 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.171909+0000", 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0846932979999999 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.172057+0000", 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000148634 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,3,5,7", 2024-09-21T10:09:15.506 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.172462+0000", 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000404977 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.176448+0000", 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003985723 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.176487+0000", 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8918000000000002e-05 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.507 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.176528+0000", 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.074e-05 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185071+0000", 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0085433290000000005 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185121+0000", 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0229000000000003e-05 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.508 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185189+0000", 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7700999999999994e-05 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185405+0000", 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000215551 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:234382 5.5 5:a51242f0:::benchmark_data_smithi032_140417_object100991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:05.111696+0000", 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 10.259464973, 2024-09-21T10:09:15.509 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0412581110000001, 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 234382 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.111696+0000", 2024-09-21T10:09:15.510 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.111696+0000", 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.111698+0000", 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2759999999999999e-06 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.111702+0000", 2024-09-21T10:09:15.511 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5709999999999999e-06 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.111717+0000", 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4640999999999999e-05 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.111725+0000", 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6510000000000001e-06 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.112535+0000", 2024-09-21T10:09:15.512 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00081080200000000001 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.112546+0000", 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0373999999999999e-05 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.144066+0000", 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.03152048 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.144207+0000", 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014109300000000001 2024-09-21T10:09:15.513 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4,6,7", 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.144571+0000", 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00036334600000000001 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.150691+0000", 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0061205119999999998 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.150751+0000", 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0093000000000002e-05 2024-09-21T10:09:15.514 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.150855+0000", 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000103292 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.150959+0000", 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010404900000000001 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.152484+0000", 2024-09-21T10:09:15.515 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0015255360000000001 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.152590+0000", 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000105555 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.152955+0000", 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00036484000000000001 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.516 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:234396 5.17 5:eb2a4a20:::benchmark_data_smithi032_140417_object101005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:05.128282+0000", 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 10.242879509, 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.05765317, 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 234396 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.517 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.128282+0000", 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.128282+0000", 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.128284+0000", 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7799999999999999e-06 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.518 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.128289+0000", 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7180000000000001e-06 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.128304+0000", 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4915e-05 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.128312+0000", 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.525e-06 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.519 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.128433+0000", 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012105 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.128443+0000", 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0316000000000001e-05 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.176563+0000", 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.048120095 2024-09-21T10:09:15.520 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.176698+0000", 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013470600000000001 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,3,5,7", 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.177153+0000", 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00045501899999999997 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185421+0000", 2024-09-21T10:09:15.521 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0082679150000000007 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185485+0000", 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4494999999999998e-05 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185624+0000", 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013860200000000001 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185694+0000", 2024-09-21T10:09:15.522 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0005999999999993e-05 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185739+0000", 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.456e-05 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185808+0000", 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9405000000000004e-05 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185935+0000", 2024-09-21T10:09:15.523 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012706300000000001 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:234407 5.17 5:ed11cc24:::benchmark_data_smithi032_140417_object101016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:05.134146+0000", 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 10.237014893, 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0642995829999999, 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:15.524 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 234407 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.134146+0000", 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.134146+0000", 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.525 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.134147+0000", 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5199999999999996e-07 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.134151+0000", 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6540000000000001e-06 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.134162+0000", 2024-09-21T10:09:15.526 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1355000000000001e-05 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.134170+0000", 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5059999999999998e-06 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.134232+0000", 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2368999999999999e-05 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:15.527 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.134241+0000", 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3230000000000004e-06 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.180300+0000", 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.046058562 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.180447+0000", 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014683099999999999 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,3,5,7", 2024-09-21T10:09:15.528 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.180884+0000", 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00043719599999999999 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.185981+0000", 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0050970579999999998 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.186017+0000", 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6393999999999997e-05 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.186060+0000", 2024-09-21T10:09:15.529 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2948999999999998e-05 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.191362+0000", 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0053013360000000002 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.198244+0000", 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0068825580000000004 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.530 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.198303+0000", 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9117999999999999e-05 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.198446+0000", 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014272199999999999 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:15.531 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:09:15.601 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:15.639 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:15.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:15.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:15.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:15.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:15.879 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:15.882 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:15 smithi045 ceph-mon[95976]: pgmap v468: 99 pgs: 3 active+remapped+backfill_wait, 5 active+remapped+backfilling, 91 active+clean; 1.7 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 7199/84627 objects misplaced (8.507%); 25 MiB/s, 406 objects/s recovering 2024-09-21T10:09:15.909 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:15.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:15 smithi032 ceph-mon[101299]: pgmap v468: 99 pgs: 3 active+remapped+backfill_wait, 5 active+remapped+backfilling, 91 active+clean; 1.7 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 7199/84627 objects misplaced (8.507%); 25 MiB/s, 406 objects/s recovering 2024-09-21T10:09:15.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:15 smithi032 ceph-mon[105748]: pgmap v468: 99 pgs: 3 active+remapped+backfill_wait, 5 active+remapped+backfilling, 91 active+clean; 1.7 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 7199/84627 objects misplaced (8.507%); 25 MiB/s, 406 objects/s recovering 2024-09-21T10:09:16.012 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:16.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:16.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:16.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:16.168 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:16.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:16.270 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:16.294 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:16.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:16.396 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:16.421 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:16.494 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:09:16.523 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:16.546 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:16.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:16.648 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:16.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.691 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:16.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:16.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:16.823 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:16.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:16.907 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[105748]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T10:09:16.907 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4108250701' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:16.907 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:16.907 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:16.908 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:16.908 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[105748]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T10:09:16.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[101299]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T10:09:16.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4108250701' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:16.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:16.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:16.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:16.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:16 smithi032 ceph-mon[101299]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T10:09:16.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:16.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:09:16.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:16.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:16 smithi045 ceph-mon[95976]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T10:09:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:16 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4108250701' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:16 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:16 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:16 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:16 smithi045 ceph-mon[95976]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T10:09:17.052 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:17.077 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:17.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:17.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:17.204 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:17.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:17.306 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:17.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:17.434 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:17.461 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:17.563 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:17.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:17.700 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:17.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:17 smithi032 ceph-mon[101299]: pgmap v470: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 4326/79152 objects misplaced (5.465%); 47 MiB/s, 758 objects/s recovering 2024-09-21T10:09:17.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:17 smithi032 ceph-mon[101299]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T10:09:17.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:17 smithi032 ceph-mon[105748]: pgmap v470: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 4326/79152 objects misplaced (5.465%); 47 MiB/s, 758 objects/s recovering 2024-09-21T10:09:17.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:17 smithi032 ceph-mon[105748]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T10:09:17.737 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:17.839 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:17.871 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:17.973 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:18.002 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:18.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:17 smithi045 ceph-mon[95976]: pgmap v470: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 4326/79152 objects misplaced (5.465%); 47 MiB/s, 758 objects/s recovering 2024-09-21T10:09:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:17 smithi045 ceph-mon[95976]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T10:09:18.103 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:18.129 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:18.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:18.231 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:18.254 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:18.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:18.356 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:18.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:18.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:18.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:18.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.508 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:18.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:18.610 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:18.643 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:18.665 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-21T10:09:18.665 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:18.665 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-21T10:09:18.665 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:18 smithi032 ceph-mon[101299]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T10:09:18.665 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-21T10:09:18.665 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:18.666 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-21T10:09:18.666 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:18 smithi032 ceph-mon[105748]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T10:09:18.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:18.774 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:18.876 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:18.898 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:18.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:19.000 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:19.024 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:19.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-21T10:09:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-21T10:09:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:18 smithi045 ceph-mon[95976]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T10:09:19.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.151 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:19.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:19.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:19.273 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:19.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:19.376 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:19.399 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:19.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:19.501 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:19.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:19.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:19.626 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:19.645 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:09:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:09:19] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.51.0" 2024-09-21T10:09:19.645 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:19 smithi032 ceph-mon[105748]: pgmap v473: 99 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 3772/78096 objects misplaced (4.830%); 37 MiB/s, 591 objects/s recovering 2024-09-21T10:09:19.645 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1810959156' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:19.645 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:19 smithi032 ceph-mon[105748]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T10:09:19.646 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:19 smithi032 ceph-mon[101299]: pgmap v473: 99 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 3772/78096 objects misplaced (4.830%); 37 MiB/s, 591 objects/s recovering 2024-09-21T10:09:19.646 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1810959156' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:19.646 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:19 smithi032 ceph-mon[101299]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T10:09:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.664 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:19.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:19.765 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:19.793 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:19.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:19.821 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:09:19.821 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:09:19.821 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:09:19.821 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:09:19.821 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:09:19.821 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:09:19.822 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:09:19.822 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:09:19.822 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:09:19.822 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:09:19.822 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:09:19.822 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:09:19.822 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:09:19.822 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:09:19.823 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:09:19.823 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:09:19.894 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:19.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:19.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:19.963 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:09:20.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:19 smithi045 ceph-mon[95976]: pgmap v473: 99 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 3772/78096 objects misplaced (4.830%); 37 MiB/s, 591 objects/s recovering 2024-09-21T10:09:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1810959156' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:19 smithi045 ceph-mon[95976]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T10:09:20.044 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:20.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:20.145 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:20.168 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:20.270 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:20.292 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:20.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:20.394 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:20.416 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:20.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:20.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:20.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:20.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:20.664 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:20.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:20.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:20.787 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:20.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:20.889 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:20.912 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:20.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:21.014 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:21.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:21.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:21.178 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:21.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:21.280 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:21.315 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:21.416 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:21.452 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:21.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:21.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:21.590 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:21.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:21.730 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:21.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:21 smithi045 ceph-mon[95976]: pgmap v476: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 97 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 1576/69693 objects misplaced (2.261%); 30 MiB/s, 473 objects/s recovering 2024-09-21T10:09:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:21 smithi045 ceph-mon[95976]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T10:09:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:21 smithi045 ceph-mon[95976]: 5.18 starting backfill to osd.6 from (0'0,0'0] MAX to 170'14023 2024-09-21T10:09:21.832 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.868 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:21.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:21.922 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:21 smithi032 ceph-mon[101299]: pgmap v476: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 97 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 1576/69693 objects misplaced (2.261%); 30 MiB/s, 473 objects/s recovering 2024-09-21T10:09:21.922 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:21 smithi032 ceph-mon[101299]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T10:09:21.922 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:21 smithi032 ceph-mon[101299]: 5.18 starting backfill to osd.6 from (0'0,0'0] MAX to 170'14023 2024-09-21T10:09:21.923 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:21 smithi032 ceph-mon[105748]: pgmap v476: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 97 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 1576/69693 objects misplaced (2.261%); 30 MiB/s, 473 objects/s recovering 2024-09-21T10:09:21.923 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:21 smithi032 ceph-mon[105748]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T10:09:21.923 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:21 smithi032 ceph-mon[105748]: 5.18 starting backfill to osd.6 from (0'0,0'0] MAX to 170'14023 2024-09-21T10:09:21.969 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:21.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:21.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:22.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:22.136 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:22.238 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:22.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:22.363 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:22.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.406 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:22.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:22.507 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:22.513 INFO:teuthology.orchestra.run.smithi032.stdout:moved 13 / 297 (4.3771%) 2024-09-21T10:09:22.513 INFO:teuthology.orchestra.run.smithi032.stdout:avg 37.125 2024-09-21T10:09:22.513 INFO:teuthology.orchestra.run.smithi032.stdout:stddev 6.31343 -> 9.57128 (expected baseline 5.69951) 2024-09-21T10:09:22.513 INFO:teuthology.orchestra.run.smithi032.stdout:min osd.5 with 25 -> 20 pgs (0.673401 -> 0.538721 * mean) 2024-09-21T10:09:22.513 INFO:teuthology.orchestra.run.smithi032.stdout:max osd.1 with 45 -> 49 pgs (1.21212 -> 1.31987 * mean) 2024-09-21T10:09:22.514 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:09:22.514 INFO:teuthology.orchestra.run.smithi032.stdout:oload 110 2024-09-21T10:09:22.514 INFO:teuthology.orchestra.run.smithi032.stdout:max_change 1 2024-09-21T10:09:22.514 INFO:teuthology.orchestra.run.smithi032.stdout:max_change_osds 4 2024-09-21T10:09:22.514 INFO:teuthology.orchestra.run.smithi032.stdout:average_utilization 0.0084 2024-09-21T10:09:22.514 INFO:teuthology.orchestra.run.smithi032.stdout:overload_utilization 0.0093 2024-09-21T10:09:22.514 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 weight 0.7057 -> 0.5285 2024-09-21T10:09:22.514 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 weight 0.8144 -> 0.6813 2024-09-21T10:09:22.514 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 weight 0.7882 -> 0.8141 2024-09-21T10:09:22.535 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:22.636 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:22.660 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:22.740 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:09:22.761 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:22.767 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:22 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:09:22.767 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:22 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'44648','4':'53352','5':'34639'}"} : dispatch 2024-09-21T10:09:22.767 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:22 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:09:22.767 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:22 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'44648','4':'53352','5':'34639'}"} : dispatch 2024-09-21T10:09:22.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:22 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:09:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:22 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'44648','4':'53352','5':'34639'}"} : dispatch 2024-09-21T10:09:22.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:22.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:22.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.915 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:22.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:23.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:23.038 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:23.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:23.140 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:23.162 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:23.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:23.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:23.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:23.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:23.387 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:23.410 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:23.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:23.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:23.542 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:23.603 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:23.629 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-21T10:09:23.629 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 110 1.0 2024-09-21T10:09:23.645 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:23.669 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:23.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:23.766 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[105748]: pgmap v478: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 694/70818 objects misplaced (0.980%); 28 MiB/s, 450 objects/s recovering 2024-09-21T10:09:23.766 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[105748]: from='client.34854 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:23.766 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:09:23.766 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'44648','4':'53352','5':'34639'}"}]': finished 2024-09-21T10:09:23.766 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[105748]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T10:09:23.766 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[101299]: pgmap v478: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 694/70818 objects misplaced (0.980%); 28 MiB/s, 450 objects/s recovering 2024-09-21T10:09:23.766 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[101299]: from='client.34854 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:23.767 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:09:23.767 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'44648','4':'53352','5':'34639'}"}]': finished 2024-09-21T10:09:23.767 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:23 smithi032 ceph-mon[101299]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T10:09:23.771 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:23 smithi045 ceph-mon[95976]: pgmap v478: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 694/70818 objects misplaced (0.980%); 28 MiB/s, 450 objects/s recovering 2024-09-21T10:09:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:23 smithi045 ceph-mon[95976]: from='client.34854 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:23 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:09:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:23 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'44648','4':'53352','5':'34639'}"}]': finished 2024-09-21T10:09:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:23 smithi045 ceph-mon[95976]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T10:09:23.793 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:23.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:23.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:23.918 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:24.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:24.042 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:24.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:24.144 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:24.166 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:24.268 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:24.294 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:24.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:24.396 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:24.423 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:24.525 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:24.562 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:24.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:24.654 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:24.664 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:24.724 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:24.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:24 smithi045 ceph-mon[95976]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T10:09:24.787 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:24 smithi032 ceph-mon[101299]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T10:09:24.788 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:24 smithi032 ceph-mon[105748]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T10:09:24.826 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:24.872 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:24.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:24.974 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:25.026 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:25.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:25.128 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:25.162 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:25.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:25.169 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:25.265 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:25.302 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:25.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:25.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:25.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:25.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:25.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:25.586 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:25.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:25.688 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:25.691 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[101299]: pgmap v481: 99 pgs: 1 peering, 1 active+remapped+backfilling, 97 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 694/69036 objects misplaced (1.005%); 25 MiB/s, 404 objects/s recovering 2024-09-21T10:09:25.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[101299]: 5.16 starting backfill to osd.6 from (0'0,0'0] MAX to 173'14046 2024-09-21T10:09:25.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[101299]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T10:09:25.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[101299]: 5.15 starting backfill to osd.3 from (0'0,0'0] MAX to 173'14120 2024-09-21T10:09:25.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[101299]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 173'15508 2024-09-21T10:09:25.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[101299]: 5.19 starting backfill to osd.1 from (0'0,0'0] MAX to 173'14153 2024-09-21T10:09:25.692 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[101299]: 5.19 starting backfill to osd.5 from (0'0,0'0] MAX to 173'14153 2024-09-21T10:09:25.692 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[105748]: pgmap v481: 99 pgs: 1 peering, 1 active+remapped+backfilling, 97 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 694/69036 objects misplaced (1.005%); 25 MiB/s, 404 objects/s recovering 2024-09-21T10:09:25.692 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[105748]: 5.16 starting backfill to osd.6 from (0'0,0'0] MAX to 173'14046 2024-09-21T10:09:25.693 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[105748]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T10:09:25.693 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[105748]: 5.15 starting backfill to osd.3 from (0'0,0'0] MAX to 173'14120 2024-09-21T10:09:25.693 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[105748]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 173'15508 2024-09-21T10:09:25.693 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[105748]: 5.19 starting backfill to osd.1 from (0'0,0'0] MAX to 173'14153 2024-09-21T10:09:25.693 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:25 smithi032 ceph-mon[105748]: 5.19 starting backfill to osd.5 from (0'0,0'0] MAX to 173'14153 2024-09-21T10:09:25.739 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:25 smithi045 ceph-mon[95976]: pgmap v481: 99 pgs: 1 peering, 1 active+remapped+backfilling, 97 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 694/69036 objects misplaced (1.005%); 25 MiB/s, 404 objects/s recovering 2024-09-21T10:09:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:25 smithi045 ceph-mon[95976]: 5.16 starting backfill to osd.6 from (0'0,0'0] MAX to 173'14046 2024-09-21T10:09:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:25 smithi045 ceph-mon[95976]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T10:09:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:25 smithi045 ceph-mon[95976]: 5.15 starting backfill to osd.3 from (0'0,0'0] MAX to 173'14120 2024-09-21T10:09:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:25 smithi045 ceph-mon[95976]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 173'15508 2024-09-21T10:09:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:25 smithi045 ceph-mon[95976]: 5.19 starting backfill to osd.1 from (0'0,0'0] MAX to 173'14153 2024-09-21T10:09:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:25 smithi045 ceph-mon[95976]: 5.19 starting backfill to osd.5 from (0'0,0'0] MAX to 173'14153 2024-09-21T10:09:25.840 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:25.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:25.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:26.029 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:26.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:26.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:26.168 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:26.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:26.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:26.330 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:26.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:26.432 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:26.475 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:26.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:26.616 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:26.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:26.718 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:26.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:26.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:26.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:26.898 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:26.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:26.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:26 smithi032 ceph-mon[101299]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T10:09:26.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:26 smithi032 ceph-mon[105748]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T10:09:27.000 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:27.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:26 smithi045 ceph-mon[95976]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T10:09:27.044 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:27.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:27.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:27.169 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:27.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:27.271 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:27.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:27.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:27.414 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:27.454 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:27.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:27.556 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:27.597 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:27.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:27.698 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:27.727 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:27.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:27.851 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:27 smithi032 ceph-mon[105748]: pgmap v484: 99 pgs: 1 active+remapped+backfill_wait, 2 remapped+peering, 2 peering, 94 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 674/65751 objects misplaced (1.025%); 10 MiB/s, 169 objects/s recovering 2024-09-21T10:09:27.851 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:27 smithi032 ceph-mon[105748]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T10:09:27.851 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:27 smithi032 ceph-mon[101299]: pgmap v484: 99 pgs: 1 active+remapped+backfill_wait, 2 remapped+peering, 2 peering, 94 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 674/65751 objects misplaced (1.025%); 10 MiB/s, 169 objects/s recovering 2024-09-21T10:09:27.851 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:27 smithi032 ceph-mon[101299]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T10:09:27.861 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:27.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:27.963 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:27.993 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:27.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:27 smithi045 ceph-mon[95976]: pgmap v484: 99 pgs: 1 active+remapped+backfill_wait, 2 remapped+peering, 2 peering, 94 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 674/65751 objects misplaced (1.025%); 10 MiB/s, 169 objects/s recovering 2024-09-21T10:09:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:27 smithi045 ceph-mon[95976]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T10:09:28.094 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:28.124 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:28.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:28.226 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:28.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:28.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:28.373 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:28.407 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:28.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:28.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:28.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.539 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:28.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:28.640 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:28.668 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:28.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:28.770 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:28.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:28.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:28.804 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:09:28.804 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:09:28.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:28.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.946 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:28.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:29.047 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:29.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.085 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:29.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:29.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:29.241 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:29.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:29.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:29.377 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:29.478 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:29.515 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:29.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:29.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:29.650 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:29.752 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:29.781 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:29.883 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:29.908 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:29.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:29.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:09:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:09:29] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.51.0" 2024-09-21T10:09:29.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:29 smithi032 ceph-mon[105748]: pgmap v486: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 2 remapped+peering, 2 peering, 93 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 926/64032 objects misplaced (1.446%); 14 MiB/s, 219 objects/s recovering 2024-09-21T10:09:29.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:29 smithi032 ceph-mon[105748]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T10:09:29.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:29 smithi032 ceph-mon[101299]: pgmap v486: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 2 remapped+peering, 2 peering, 93 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 926/64032 objects misplaced (1.446%); 14 MiB/s, 219 objects/s recovering 2024-09-21T10:09:29.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:29 smithi032 ceph-mon[101299]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T10:09:30.009 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:30.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.033 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:30.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:30.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:29 smithi045 ceph-mon[95976]: pgmap v486: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 2 remapped+peering, 2 peering, 93 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 926/64032 objects misplaced (1.446%); 14 MiB/s, 219 objects/s recovering 2024-09-21T10:09:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:29 smithi045 ceph-mon[95976]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T10:09:30.134 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:30.169 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:30.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:30.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:30.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.302 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:30.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:30.351 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:30.403 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:30.427 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:30.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:30.528 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:30.552 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:30.654 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:30.695 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:30.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:30.797 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:30.833 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:30.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:30.935 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:30.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:30 smithi032 ceph-mon[101299]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T10:09:30.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:30.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:30.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:30 smithi032 ceph-mon[101299]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T10:09:30.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:30 smithi032 ceph-mon[105748]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T10:09:30.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:30.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:30.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:30 smithi032 ceph-mon[105748]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T10:09:30.972 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:30.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:30 smithi045 ceph-mon[95976]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T10:09:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:30 smithi045 ceph-mon[95976]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T10:09:31.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:31.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.103 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:31.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:31.204 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.258 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:31.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:31.359 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:31.387 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:31.488 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:31.515 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:31.565 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:09:31.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:31.642 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:31.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:31.744 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:31.773 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:31.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:31.875 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.906 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:31.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:32.007 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:32.042 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:32.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:32.144 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:32.179 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:32.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[101299]: pgmap v489: 99 pgs: 2 peering, 2 active+remapped+backfilling, 95 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 1181/53868 objects misplaced (2.192%); 25 MiB/s, 394 objects/s recovering 2024-09-21T10:09:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:09:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:09:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3004769552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:32.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[105748]: pgmap v489: 99 pgs: 2 peering, 2 active+remapped+backfilling, 95 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 1181/53868 objects misplaced (2.192%); 25 MiB/s, 394 objects/s recovering 2024-09-21T10:09:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:09:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:09:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3004769552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:31 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:32.281 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:31 smithi045 ceph-mon[95976]: pgmap v489: 99 pgs: 2 peering, 2 active+remapped+backfilling, 95 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 1181/53868 objects misplaced (2.192%); 25 MiB/s, 394 objects/s recovering 2024-09-21T10:09:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:09:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:09:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:31 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3004769552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:31 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:32.333 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:32.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:32.435 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:32.462 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:32.567 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:32.601 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:32.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:32.693 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:09:32.703 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:32.734 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:32.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:32.836 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:32.862 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:32.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:32.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:32 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:32.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:32 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:09:32.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:32 smithi032 ceph-mon[101299]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T10:09:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:32 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:32 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:09:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:32 smithi032 ceph-mon[105748]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T10:09:32.964 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:32.986 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:32.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:33.087 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.112 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:33.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:33.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:33.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.237 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:33.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:33.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:32 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:33.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:32 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:09:33.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:32 smithi045 ceph-mon[95976]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T10:09:33.339 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:33.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.363 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:33.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:33.464 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:33.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.488 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:33.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:33.589 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.613 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:33.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:33.714 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:33.738 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:33.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:33.839 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:33.869 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:33.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:33.971 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:33.986 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:33 smithi032 ceph-mon[101299]: pgmap v491: 99 pgs: 2 peering, 1 active+remapped+backfilling, 96 active+clean; 1.1 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 929/51900 objects misplaced (1.790%); 23 MiB/s, 369 objects/s recovering 2024-09-21T10:09:33.986 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:33.986 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:33 smithi032 ceph-mon[101299]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T10:09:33.986 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:33 smithi032 ceph-mon[105748]: pgmap v491: 99 pgs: 2 peering, 1 active+remapped+backfilling, 96 active+clean; 1.1 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 929/51900 objects misplaced (1.790%); 23 MiB/s, 369 objects/s recovering 2024-09-21T10:09:33.986 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:33.986 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:33 smithi032 ceph-mon[105748]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T10:09:34.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.009 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:34.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:34.110 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:34.174 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:34.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:34.275 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:33 smithi045 ceph-mon[95976]: pgmap v491: 99 pgs: 2 peering, 1 active+remapped+backfilling, 96 active+clean; 1.1 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 929/51900 objects misplaced (1.790%); 23 MiB/s, 369 objects/s recovering 2024-09-21T10:09:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:33 smithi045 ceph-mon[95976]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T10:09:34.305 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:34.407 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:34.437 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:34.539 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:34.570 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:34.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:34.672 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:34.691 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:34.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:34.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:34.811 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:34.860 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:34.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:34.947 INFO:teuthology.orchestra.run.smithi032.stdout:moved 10 / 297 (3.367%) 2024-09-21T10:09:34.948 INFO:teuthology.orchestra.run.smithi032.stdout:avg 37.125 2024-09-21T10:09:34.948 INFO:teuthology.orchestra.run.smithi032.stdout:stddev 9.45301 -> 11.5913 (expected baseline 5.69951) 2024-09-21T10:09:34.948 INFO:teuthology.orchestra.run.smithi032.stdout:min osd.5 with 21 -> 17 pgs (0.565657 -> 0.457912 * mean) 2024-09-21T10:09:34.948 INFO:teuthology.orchestra.run.smithi032.stdout:max osd.1 with 50 -> 53 pgs (1.3468 -> 1.42761 * mean) 2024-09-21T10:09:34.948 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:09:34.948 INFO:teuthology.orchestra.run.smithi032.stdout:oload 110 2024-09-21T10:09:34.949 INFO:teuthology.orchestra.run.smithi032.stdout:max_change 1 2024-09-21T10:09:34.949 INFO:teuthology.orchestra.run.smithi032.stdout:max_change_osds 4 2024-09-21T10:09:34.949 INFO:teuthology.orchestra.run.smithi032.stdout:average_utilization 0.0068 2024-09-21T10:09:34.949 INFO:teuthology.orchestra.run.smithi032.stdout:overload_utilization 0.0075 2024-09-21T10:09:34.949 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 weight 0.5285 -> 0.3914 2024-09-21T10:09:34.949 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 weight 0.6813 -> 0.5625 2024-09-21T10:09:34.949 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 weight 0.8141 -> 0.8385 2024-09-21T10:09:34.962 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:34.988 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:35.090 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:35.112 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:35.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'36863','4':'54951','5':'25650'}"} : dispatch 2024-09-21T10:09:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'36863','4':'54951','5':'25650'}"} : dispatch 2024-09-21T10:09:35.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:35.236 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:35.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'36863','4':'54951','5':'25650'}"} : dispatch 2024-09-21T10:09:35.338 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.361 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:35.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:35.462 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:35.485 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:35.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:35.587 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:35.609 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:35.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:35.733 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:35.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:35.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:09:35.836 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:35.859 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:35.961 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:35.983 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:36.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:36.109 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:36.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:35 smithi032 ceph-mon[101299]: pgmap v494: 99 pgs: 3 peering, 96 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 23 MiB/s, 366 objects/s recovering 2024-09-21T10:09:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:35 smithi032 ceph-mon[101299]: from='client.34860 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:35 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'36863','4':'54951','5':'25650'}"}]': finished 2024-09-21T10:09:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:35 smithi032 ceph-mon[101299]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T10:09:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:35 smithi032 ceph-mon[105748]: pgmap v494: 99 pgs: 3 peering, 96 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 23 MiB/s, 366 objects/s recovering 2024-09-21T10:09:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:35 smithi032 ceph-mon[105748]: from='client.34860 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:35 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'36863','4':'54951','5':'25650'}"}]': finished 2024-09-21T10:09:36.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:35 smithi032 ceph-mon[105748]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T10:09:36.211 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:36.233 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:36.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:35 smithi045 ceph-mon[95976]: pgmap v494: 99 pgs: 3 peering, 96 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 23 MiB/s, 366 objects/s recovering 2024-09-21T10:09:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:35 smithi045 ceph-mon[95976]: from='client.34860 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:35 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'36863','4':'54951','5':'25650'}"}]': finished 2024-09-21T10:09:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:35 smithi045 ceph-mon[95976]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T10:09:36.334 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:36.360 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:36.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:36.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:36.485 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:36.586 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:36.610 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:36.712 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.735 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:36.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:36.836 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:36.863 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:36.965 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:36.993 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:36.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:37.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:37.124 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:36 smithi032 ceph-mon[105748]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T10:09:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:36 smithi032 ceph-mon[101299]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T10:09:37.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:37.265 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:37.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:37.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:36 smithi045 ceph-mon[95976]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T10:09:37.342 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:09:37.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 1000 3.0 2024-09-21T10:09:37.366 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:37.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.456 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:37.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:37.557 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:37.588 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:37.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:37.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:37.745 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:37.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:37.846 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:09:37.847 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:09:37.847 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:37.878 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:37.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:37.891 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:09:37.891 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:09:37.931 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:09:37.932 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:09:37.971 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:09:37.971 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:09:37.981 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:38.008 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:38.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:38.016 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:09:38.017 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:09:38.050 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:09:38.051 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:09:38.083 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:09:38.083 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:09:38.110 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:38.123 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[101299]: pgmap v497: 99 pgs: 1 peering, 98 active+clean; 903 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 13 MiB/s, 213 objects/s recovering 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[101299]: 5.13 starting backfill to osd.4 from (0'0,0'0] MAX to 183'14420 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[101299]: 5.3 starting backfill to osd.1 from (0'0,0'0] MAX to 183'14427 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[101299]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[101299]: 5.e starting backfill to osd.3 from (0'0,0'0] MAX to 183'15867 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3886515107' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[105748]: pgmap v497: 99 pgs: 1 peering, 98 active+clean; 903 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 13 MiB/s, 213 objects/s recovering 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[105748]: 5.13 starting backfill to osd.4 from (0'0,0'0] MAX to 183'14420 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[105748]: 5.3 starting backfill to osd.1 from (0'0,0'0] MAX to 183'14427 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[105748]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[105748]: 5.e starting backfill to osd.3 from (0'0,0'0] MAX to 183'15867 2024-09-21T10:09:38.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:37 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3886515107' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:38.126 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:09:38.126 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:09:38.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:38.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:38.250 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:37 smithi045 ceph-mon[95976]: pgmap v497: 99 pgs: 1 peering, 98 active+clean; 903 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 13 MiB/s, 213 objects/s recovering 2024-09-21T10:09:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:37 smithi045 ceph-mon[95976]: 5.13 starting backfill to osd.4 from (0'0,0'0] MAX to 183'14420 2024-09-21T10:09:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:37 smithi045 ceph-mon[95976]: 5.3 starting backfill to osd.1 from (0'0,0'0] MAX to 183'14427 2024-09-21T10:09:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:37 smithi045 ceph-mon[95976]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T10:09:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:37 smithi045 ceph-mon[95976]: 5.e starting backfill to osd.3 from (0'0,0'0] MAX to 183'15867 2024-09-21T10:09:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:37 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3886515107' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:38.293 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:38.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:38.394 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:38.417 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:38.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:38.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:38.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:38.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:38.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:38.674 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:38.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:38.776 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:38.806 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:38.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:38.911 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:38.953 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:38.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:39.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:39.097 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:39.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:39.200 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:39.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.236 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:39.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:39.337 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:39.377 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:39.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:39.479 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:39.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:39.627 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:39.667 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:39.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:39.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.797 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:39.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:39.898 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:39.931 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:39.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:09:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:09:39] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.51.0" 2024-09-21T10:09:40.033 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:40.073 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:40.113 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:40.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:40.202 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:40.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:40.265 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:39 smithi032 ceph-mon[105748]: pgmap v499: 99 pgs: 2 remapped+peering, 1 peering, 96 active+clean; 878 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 527 op/s 2024-09-21T10:09:40.266 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:39 smithi032 ceph-mon[101299]: pgmap v499: 99 pgs: 2 remapped+peering, 1 peering, 96 active+clean; 878 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 527 op/s 2024-09-21T10:09:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:39 smithi045 ceph-mon[95976]: pgmap v499: 99 pgs: 2 remapped+peering, 1 peering, 96 active+clean; 878 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 527 op/s 2024-09-21T10:09:40.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:40.329 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:40.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:40.453 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:40.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:40.577 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:40.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:40.679 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:40.701 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:40.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:40.825 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:40.927 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:40.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:41.052 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:41.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.087 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:41.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:41.188 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:41.216 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:41.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:40 smithi045 ceph-mon[95976]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T10:09:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:40 smithi045 ceph-mon[95976]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T10:09:41.317 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:41.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:41.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:41.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:40 smithi032 ceph-mon[101299]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:40 smithi032 ceph-mon[101299]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:40 smithi032 ceph-mon[105748]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:40 smithi032 ceph-mon[105748]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T10:09:41.463 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:41.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:41.565 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:41.584 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:41.587 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:41.689 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:41.712 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:41.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:41.814 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:41.838 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:41.939 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:41.963 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:41.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:42.064 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:42.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:42.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:42.162 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:41 smithi032 ceph-mon[105748]: pgmap v501: 99 pgs: 1 active+clean+remapped, 2 remapped+peering, 96 active+clean; 742 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 4.3 MiB/s, 69 objects/s recovering 2024-09-21T10:09:42.163 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:41 smithi032 ceph-mon[105748]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T10:09:42.163 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:41 smithi032 ceph-mon[101299]: pgmap v501: 99 pgs: 1 active+clean+remapped, 2 remapped+peering, 96 active+clean; 742 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 4.3 MiB/s, 69 objects/s recovering 2024-09-21T10:09:42.163 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:41 smithi032 ceph-mon[101299]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T10:09:42.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.239 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:42.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:41 smithi045 ceph-mon[95976]: pgmap v501: 99 pgs: 1 active+clean+remapped, 2 remapped+peering, 96 active+clean; 742 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 4.3 MiB/s, 69 objects/s recovering 2024-09-21T10:09:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:41 smithi045 ceph-mon[95976]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T10:09:42.336 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:09:42.336 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:09:42.336 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:09:42.336 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:09:42.336 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:09:42.340 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:42.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:42.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:42.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:42.523 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:42.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:42.625 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:42.663 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:42.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:42.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:42.806 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:42.908 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:42.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:42.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:43.053 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:43.088 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:43.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:43.190 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:43.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:42 smithi032 ceph-mon[101299]: pgmap v504: 99 pgs: 1 peering, 1 active+clean+remapped, 97 active+clean; 702 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 17 MiB/s, 272 objects/s recovering 2024-09-21T10:09:43.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:42 smithi032 ceph-mon[105748]: pgmap v504: 99 pgs: 1 peering, 1 active+clean+remapped, 97 active+clean; 702 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 17 MiB/s, 272 objects/s recovering 2024-09-21T10:09:43.223 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:43.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:42 smithi045 ceph-mon[95976]: pgmap v504: 99 pgs: 1 peering, 1 active+clean+remapped, 97 active+clean; 702 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 17 MiB/s, 272 objects/s recovering 2024-09-21T10:09:43.326 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:43.354 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:43.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:43.455 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:43.498 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:43.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:43.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:09:43.600 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:43.678 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:43.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:43.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:43.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.830 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:43.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:43.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:43.959 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:43.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:44.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:44.086 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:44.188 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:44.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.231 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:44.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:44.333 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:44.376 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:44.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:44.478 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:44.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:44.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:44.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:44.654 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:44.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:44.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.794 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:44.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:44.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:44.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.938 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:44.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:45.039 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:45.071 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:45.173 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:45.203 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:45.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:45.305 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:45.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.351 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:45.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:45.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:45.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:45 smithi032 ceph-mon[101299]: pgmap v505: 99 pgs: 1 peering, 1 active+clean+remapped, 97 active+clean; 647 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 15 MiB/s, 240 objects/s recovering 2024-09-21T10:09:45.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:45 smithi032 ceph-mon[105748]: pgmap v505: 99 pgs: 1 peering, 1 active+clean+remapped, 97 active+clean; 647 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 15 MiB/s, 240 objects/s recovering 2024-09-21T10:09:45.484 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:45.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:45.585 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:45.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.628 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:45.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:45.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:45.771 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:45.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:45 smithi045 ceph-mon[95976]: pgmap v505: 99 pgs: 1 peering, 1 active+clean+remapped, 97 active+clean; 647 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 15 MiB/s, 240 objects/s recovering 2024-09-21T10:09:45.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:09:45.873 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:45.933 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:46.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:46.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.076 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:46.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:46.177 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:46.208 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:46.309 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.350 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:46.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:46.451 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:46.484 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:46.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:46.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:46.612 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:46.714 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:46.740 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:46.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:46.842 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:46.866 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:46.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.867 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:46.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:46.924 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:46.969 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:46.996 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:46.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout:moved 1 / 297 (0.3367%) 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout:avg 37.125 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout:stddev 11.5913 -> 11.559 (expected baseline 5.69951) 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout:min osd.5 with 17 -> 17 pgs (0.457912 -> 0.457912 * mean) 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout:max osd.1 with 53 -> 53 pgs (1.42761 -> 1.42761 * mean) 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout:oload 1000 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout:max_change 3 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout:max_change_osds 4 2024-09-21T10:09:47.026 INFO:teuthology.orchestra.run.smithi032.stdout:average_utilization 0.0043 2024-09-21T10:09:47.027 INFO:teuthology.orchestra.run.smithi032.stdout:overload_utilization 0.0432 2024-09-21T10:09:47.027 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 weight 0.8385 -> 0.8801 2024-09-21T10:09:47.027 INFO:teuthology.orchestra.run.smithi032.stderr:no change 2024-09-21T10:09:47.097 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:47.124 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:47.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:47.226 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:47.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:47.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:47.306 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:09:47.307 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:09:47 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T10:09:47.019+0000 7fce75a2e640 -1 mgr.server reply reply (1) Operation not permitted no change 2024-09-21T10:09:47.362 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:47.384 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:47.486 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:47.523 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:47.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:47.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:47.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:47.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:47 smithi032 ceph-mon[105748]: pgmap v506: 99 pgs: 1 peering, 98 active+clean; 526 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 11 MiB/s, 171 objects/s recovering 2024-09-21T10:09:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:47 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2017882235' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:47 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:47.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:47 smithi032 ceph-mon[101299]: pgmap v506: 99 pgs: 1 peering, 98 active+clean; 526 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 11 MiB/s, 171 objects/s recovering 2024-09-21T10:09:47.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:47 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2017882235' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:47.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:47 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:47.768 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:47 smithi045 ceph-mon[95976]: pgmap v506: 99 pgs: 1 peering, 98 active+clean; 526 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 11 MiB/s, 171 objects/s recovering 2024-09-21T10:09:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:47 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2017882235' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:47 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.804 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:47.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:47.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:47.946 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:47.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:48.048 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:48.088 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:48.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:48.189 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:48.226 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:48.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:48.328 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:48.376 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:48.478 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:48.509 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:48.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:48.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:48.652 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:48.679 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[105748]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:48.679 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:09:48.679 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:09:48.679 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:48.679 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[105748]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T10:09:48.679 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[105748]: 5.13 scrub starts 2024-09-21T10:09:48.679 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[105748]: 5.13 scrub ok 2024-09-21T10:09:48.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[101299]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:48.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:09:48.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:09:48.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:48.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[101299]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T10:09:48.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[101299]: 5.13 scrub starts 2024-09-21T10:09:48.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:48 smithi032 ceph-mon[101299]: 5.13 scrub ok 2024-09-21T10:09:48.755 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:48.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:48 smithi045 ceph-mon[95976]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:09:48.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:48 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:09:48.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:48 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:09:48.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:48 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:48.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:48 smithi045 ceph-mon[95976]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T10:09:48.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:48 smithi045 ceph-mon[95976]: 5.13 scrub starts 2024-09-21T10:09:48.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:48 smithi045 ceph-mon[95976]: 5.13 scrub ok 2024-09-21T10:09:48.796 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:48.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:48.898 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:48.940 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:49.042 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:49.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.082 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:49.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:49.183 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:49.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:49.325 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:49.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:49.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:49.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:49.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:49.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:49.598 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:49.635 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:49.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:49.658 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:49 smithi032 ceph-mon[105748]: pgmap v508: 99 pgs: 99 active+clean; 458 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:09:49.658 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:49.658 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:49.659 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:49 smithi032 ceph-mon[101299]: pgmap v508: 99 pgs: 99 active+clean; 458 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:09:49.659 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:49.659 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:49.659 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:09:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:09:49] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.51.0" 2024-09-21T10:09:49.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:49.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:49.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:49 smithi045 ceph-mon[95976]: pgmap v508: 99 pgs: 99 active+clean; 458 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:09:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:09:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:49.876 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:49.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:49.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:50.018 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:50.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.052 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:50.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:50.153 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.193 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:50.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:50.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:50.325 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:50.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:50.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:50.456 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:50.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:50.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:50.588 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:50.690 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:50.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:50.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:50.823 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:50.859 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:50.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:50.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:50.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.000 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:51.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:51.102 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:51.131 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:51.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:51.233 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.270 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:51.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:51.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:51.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.403 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:51.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:51.481 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:09:51.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 1000 3.0 2024-09-21T10:09:51.503 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:51.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:51.635 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:51.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.683 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:51.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:51.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:51 smithi032 ceph-mon[105748]: pgmap v509: 99 pgs: 99 active+clean; 207 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 957 op/s 2024-09-21T10:09:51.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:51 smithi032 ceph-mon[101299]: pgmap v509: 99 pgs: 99 active+clean; 207 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 957 op/s 2024-09-21T10:09:51.707 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Removed 133388 objects 2024-09-21T10:09:51.707 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Clean up completed and total clean up time :171.723 2024-09-21T10:09:51.785 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:51 smithi045 ceph-mon[95976]: pgmap v509: 99 pgs: 99 active+clean; 207 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 957 op/s 2024-09-21T10:09:51.807 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:51.909 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:51.930 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:51.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:52.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:52.055 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:52.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:52.156 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:52.178 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:52.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:52.280 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:52.280 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-21T10:09:52.281 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-21T10:09:52.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:52.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:52.406 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:52.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:52.531 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:52.555 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:52.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:52.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:52.682 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:52.784 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:52.806 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:52.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:52.908 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:52.935 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:52.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:53.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:53.058 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:53.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:53.160 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:53.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:53.193 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:09:53.193 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:09:53.193 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:09:53.193 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207612 5.6 5:62ed9bd4:::benchmark_data_smithi032_140417_object74221:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:27.040775+0000", 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 86.143669838999998, 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.84635733999999996, 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.194 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207612 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040775+0000", 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040775+0000", 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.195 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040777+0000", 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.968e-06 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040784+0000", 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9439999999999999e-06 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040808+0000", 2024-09-21T10:09:53.196 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3889000000000001e-05 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040819+0000", 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0879e-05 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.104893+0000", 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.064074148999999997 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.104902+0000", 2024-09-21T10:09:53.197 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9360000000000005e-06 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.887028+0000", 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.78212554999999995 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.887133+0000", 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000105025 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.198 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207616 5.8 5:130c89fe:::benchmark_data_smithi032_140417_object74225:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:27.040836+0000", 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 86.143608912999994, 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.86514601400000002, 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207616 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.199 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040836+0000", 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040836+0000", 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040837+0000", 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9100000000000004e-07 2024-09-21T10:09:53.200 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040839+0000", 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3530000000000002e-06 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040846+0000", 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2400000000000001e-06 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.201 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040852+0000", 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.947e-06 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.121149+0000", 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.080296815999999993 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.121177+0000", 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.8121999999999999e-05 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.202 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.905869+0000", 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.78469153599999997 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.905982+0000", 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000113509 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207619 5.18 5:1f63d48b:::benchmark_data_smithi032_140417_object74228:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2024-09-21T10:09:53.203 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:27.040859+0000", 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 86.143586271999993, 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.89001014199999995, 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207619 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040859+0000", 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040859+0000", 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040859+0000", 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9400000000000001e-07 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040861+0000", 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9599999999999999e-06 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.205 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040867+0000", 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0979999999999999e-06 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040873+0000", 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4600000000000002e-06 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.120926+0000", 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.080052940000000003 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.206 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.120935+0000", 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9679999999999995e-06 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.930781+0000", 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.80984641599999996 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.930869+0000", 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7905999999999995e-05 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.207 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207621 5.16 5:6a053a4e:::benchmark_data_smithi032_140417_object74230:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:27.040878+0000", 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 86.143566961000005, 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.87456720300000002, 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207621 2024-09-21T10:09:53.208 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040878+0000", 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040878+0000", 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040879+0000", 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.7e-07 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.209 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040880+0000", 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7570000000000001e-06 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040886+0000", 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1489999999999999e-06 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040892+0000", 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0810000000000001e-06 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.210 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.104922+0000", 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.064030835999999994 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.104928+0000", 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8320000000000002e-06 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.915309+0000", 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.81038076800000003 2024-09-21T10:09:53.211 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.915445+0000", 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013641 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:207629 5.18 5:1ad30e98:::benchmark_data_smithi032_140417_object74238:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:27.040897+0000", 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 86.143548480999996, 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.89570315, 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.212 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 207629 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040897+0000", 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.213 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040897+0000", 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040897+0000", 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.58e-07 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040899+0000", 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7540000000000001e-06 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.214 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040905+0000", 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0039999999999996e-06 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.040910+0000", 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9119999999999997e-06 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.120957+0000", 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.080047459000000001 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.215 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.120963+0000", 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3589999999999996e-06 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.931403+0000", 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.81043927999999998 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.931532+0000", 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012945400000000001 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.216 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.931931+0000", 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00039907899999999999 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.934026+0000", 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0020951300000000002 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.935628+0000", 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0016017360000000001 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.217 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.936518+0000", 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00088975000000000002 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.936576+0000", 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7976000000000002e-05 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:27.936600+0000", 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3898999999999999e-05 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.218 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:234361 5.b 5:d218a76c:::benchmark_data_smithi032_140417_object100970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:03.975817+0000", 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 49.208628265999998, 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2734688080000001, 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 234361 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.219 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:03.975817+0000", 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:03.975817+0000", 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:03.975818+0000", 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5460000000000001e-06 2024-09-21T10:09:53.220 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:03.975824+0000", 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.631e-06 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:03.975837+0000", 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2663000000000001e-05 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:03.975846+0000", 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8689999999999993e-06 2024-09-21T10:09:53.221 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:03.975907+0000", 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1095999999999999e-05 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:03.975917+0000", 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.9119999999999993e-06 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:04.973376+0000", 2024-09-21T10:09:53.222 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.997459023 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:04.973393+0000", 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7750000000000001e-05 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.210288+0000", 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.236894538 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.223 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.210551+0000", 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00026342 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5,7", 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.211402+0000", 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00085107800000000001 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.229038+0000", 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.017635983000000001 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.224 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.229053+0000", 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4314999999999999e-05 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.229095+0000", 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2119000000000001e-05 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249128+0000", 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.020033511 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.225 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249180+0000", 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1189000000000001e-05 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249286+0000", 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000106165 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:234378 5.b 5:d8b2a35f:::benchmark_data_smithi032_140417_object100987:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:09:53.226 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:05.110262+0000", 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 48.074183292999997, 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.139303357, 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 234378 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.110262+0000", 2024-09-21T10:09:53.227 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.110262+0000", 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.110263+0000", 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.229e-06 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.228 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.110268+0000", 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8319999999999996e-06 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.110280+0000", 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1644000000000001e-05 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.110287+0000", 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.17e-06 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.110353+0000", 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6150999999999997e-05 2024-09-21T10:09:53.229 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.110363+0000", 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0311e-05 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.229133+0000", 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.118770117 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.229228+0000", 2024-09-21T10:09:53.230 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4514999999999999e-05 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5,7", 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.229367+0000", 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013871700000000001 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249034+0000", 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.019667882000000001 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249080+0000", 2024-09-21T10:09:53.231 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5821000000000001e-05 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249328+0000", 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000247517 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249411+0000", 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3745999999999994e-05 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.232 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249464+0000", 2024-09-21T10:09:53.233 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2701999999999998e-05 2024-09-21T10:09:53.233 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249565+0000", 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000101003 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:234417 5.b 5:debe7d34:::benchmark_data_smithi032_140417_object101026:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:05.143361+0000", 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 48.041084034000001, 2024-09-21T10:09:53.234 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.142473053, 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 234417 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.143361+0000", 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.235 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.143361+0000", 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.143363+0000", 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7540000000000001e-06 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.143368+0000", 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6809999999999997e-06 2024-09-21T10:09:53.236 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.143381+0000", 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3556e-05 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.143389+0000", 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6249999999999998e-06 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.143459+0000", 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0366999999999999e-05 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:05.143470+0000", 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0519e-05 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.234286+0000", 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0908168300000001 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.238 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.235956+0000", 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0016699289999999999 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5,7", 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.236386+0000", 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00042982300000000002 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249041+0000", 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012654889000000001 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.239 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249375+0000", 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000333808 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.249606+0000", 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00023098199999999999 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.285667+0000", 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.036060863999999998 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.240 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.285723+0000", 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6029000000000002e-05 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:06.285834+0000", 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000111397 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.241 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241767 5.b 5:dd8b8b4d:::benchmark_data_smithi032_140417_object108376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.559871+0000", 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.624573732000002, 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.97648648699999996, 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241767 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.242 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559871+0000", 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559871+0000", 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559873+0000", 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9580000000000001e-06 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.243 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559879+0000", 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1490000000000002e-06 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559891+0000", 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2670999999999999e-05 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559899+0000", 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0600000000000008e-06 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.244 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559984+0000", 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4837000000000001e-05 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.560121+0000", 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013717499999999999 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5,7", 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.560241+0000", 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000119852 2024-09-21T10:09:53.245 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.567391+0000", 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0071502320000000003 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.568018+0000", 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00062714499999999996 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.577631+0000", 2024-09-21T10:09:53.246 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0096120939999999998 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585548+0000", 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0079174420000000002 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585561+0000", 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3477e-05 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.247 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536278+0000", 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95071670500000005 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536358+0000", 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9690000000000004e-05 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241780 5.b 5:dbce6705:::benchmark_data_smithi032_140417_object108389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.578234+0000", 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.606210875999999, 2024-09-21T10:09:53.248 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95871411699999998, 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241780 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578234+0000", 2024-09-21T10:09:53.249 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578234+0000", 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578236+0000", 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8279999999999999e-06 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578241+0000", 2024-09-21T10:09:53.250 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3630000000000002e-06 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578258+0000", 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6198999999999999e-05 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578266+0000", 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1899999999999995e-06 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.251 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578403+0000", 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013736600000000001 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578564+0000", 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016075700000000001 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5,7", 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578692+0000", 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000127663 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.252 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585713+0000", 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0070217600000000002 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585722+0000", 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4800000000000001e-06 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536878+0000", 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95115663500000003 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.253 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536948+0000", 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9876000000000007e-05 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241783 5.b 5:d9813225:::benchmark_data_smithi032_140417_object108392:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.583475+0000", 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.600969878999997, 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.96363426399999996, 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.254 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241783 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583475+0000", 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.255 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583475+0000", 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583477+0000", 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2069999999999998e-06 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583483+0000", 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.835e-06 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583500+0000", 2024-09-21T10:09:53.256 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6920999999999999e-05 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583508+0000", 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6650000000000003e-06 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585752+0000", 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0022445170000000001 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.257 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585760+0000", 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9240000000000007e-06 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536976+0000", 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95121591100000003 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.537083+0000", 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000107181 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.258 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.537170+0000", 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6984000000000006e-05 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.546792+0000", 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0096215229999999999 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.546849+0000", 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7135000000000001e-05 2024-09-21T10:09:53.259 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.546981+0000", 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013146699999999999 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547069+0000", 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8830000000000002e-05 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547110+0000", 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0163999999999997e-05 2024-09-21T10:09:53.260 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241790 5.b 5:d9f071b7:::benchmark_data_smithi032_140417_object108399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.588776+0000", 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.595669266999998, 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.98105678799999996, 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241790 2024-09-21T10:09:53.261 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588776+0000", 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588776+0000", 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588777+0000", 2024-09-21T10:09:53.262 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0750000000000001e-06 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588781+0000", 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4530000000000004e-06 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588796+0000", 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4842e-05 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588804+0000", 2024-09-21T10:09:53.263 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9899999999999997e-06 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588920+0000", 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011530400000000001 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588930+0000", 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0526999999999999e-05 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.542310+0000", 2024-09-21T10:09:53.264 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95338037600000003 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.542440+0000", 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000129794 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.542551+0000", 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000110707 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.546886+0000", 2024-09-21T10:09:53.265 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0043350580000000001 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560416+0000", 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013530333 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569756+0000", 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0093396910000000007 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.266 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569791+0000", 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4666999999999999e-05 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569833+0000", 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1971000000000002e-05 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241804 5.b 5:d56f6689:::benchmark_data_smithi032_140417_object108413:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.604378+0000", 2024-09-21T10:09:53.267 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.580067477999997, 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.965583466, 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241804 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604378+0000", 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.268 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604378+0000", 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604379+0000", 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.207e-06 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604384+0000", 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1599999999999997e-06 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.269 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604394+0000", 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4900000000000006e-06 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604400+0000", 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2310000000000001e-06 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604454+0000", 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3767000000000002e-05 2024-09-21T10:09:53.270 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604462+0000", 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1610000000000003e-06 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547153+0000", 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.94269127799999997 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547257+0000", 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010452 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.271 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547378+0000", 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000120383 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560454+0000", 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013076119000000001 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560492+0000", 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8175e-05 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.272 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569891+0000", 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0093986539999999993 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569939+0000", 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7734000000000001e-05 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569961+0000", 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2586999999999999e-05 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.273 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241832 5.b 5:dd944995:::benchmark_data_smithi032_140417_object108441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.636813+0000", 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.547631674999998, 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.93322067200000003, 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241832 2024-09-21T10:09:53.274 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636813+0000", 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636813+0000", 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.275 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636815+0000", 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3859999999999999e-06 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636820+0000", 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9359999999999998e-06 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636836+0000", 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6711e-05 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.276 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636845+0000", 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0560000000000005e-06 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.637012+0000", 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016762100000000001 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.637022+0000", 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3950000000000007e-06 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.277 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.551757+0000", 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.91473561999999997 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.551886+0000", 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012900699999999999 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.552039+0000", 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000152597 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.278 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560541+0000", 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0085025929999999993 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569747+0000", 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0092056529999999994 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569970+0000", 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000222554 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.279 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570012+0000", 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2849000000000003e-05 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570034+0000", 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1693999999999999e-05 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.280 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241877 5.b 5:d8095acf:::benchmark_data_smithi032_140417_object108486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.682983+0000", 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.501462046, 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.88718399199999998, 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241877 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.281 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.682983+0000", 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.682983+0000", 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.682984+0000", 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8999999999999995e-07 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.282 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.682987+0000", 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.569e-06 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.683033+0000", 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5096000000000003e-05 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.683042+0000", 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2639999999999997e-06 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.283 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.684225+0000", 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001183403 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.684234+0000", 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5529999999999993e-06 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.555793+0000", 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.87155956400000001 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.284 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.555895+0000", 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000101887 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.556014+0000", 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000118559 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560577+0000", 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.004563587 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.285 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570044+0000", 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0094666330000000003 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570101+0000", 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6842999999999998e-05 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570130+0000", 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.9115000000000002e-05 2024-09-21T10:09:53.286 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570167+0000", 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.7129000000000003e-05 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241888 5.b 5:dd26de34:::benchmark_data_smithi032_140417_object108497:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.695983+0000", 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.488462175999999, 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.88328308200000005, 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.287 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241888 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.695983+0000", 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.288 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.695983+0000", 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.695984+0000", 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2759999999999999e-06 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.695989+0000", 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0289999999999996e-06 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.289 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.696000+0000", 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0343000000000001e-05 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.696007+0000", 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3010000000000001e-06 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.696305+0000", 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029825899999999999 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.290 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.696314+0000", 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3909999999999998e-06 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560605+0000", 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.86429099799999998 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560720+0000", 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011551499999999999 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.291 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560837+0000", 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011686 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569879+0000", 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0090424839999999999 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570052+0000", 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000172614 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.292 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579189+0000", 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0091371950000000007 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579222+0000", 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2437000000000001e-05 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579266+0000", 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4379999999999999e-05 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.293 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241903 5.b 5:d931b21c:::benchmark_data_smithi032_140417_object108512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.712698+0000", 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.471746938999999, 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.886554913, 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.294 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241903 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712698+0000", 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712698+0000", 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.295 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712700+0000", 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.477e-06 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712704+0000", 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4549999999999997e-06 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712796+0000", 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1836999999999995e-05 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.296 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712806+0000", 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0049e-05 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712907+0000", 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000101099 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712916+0000", 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9979999999999992e-06 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.297 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.565163+0000", 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.85224674199999995 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.565269+0000", 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010645900000000001 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.565386+0000", 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000116946 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.298 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570225+0000", 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0048383999999999996 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570242+0000", 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7431e-05 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599133+0000", 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.028890592 2024-09-21T10:09:53.299 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599193+0000", 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9954000000000002e-05 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599253+0000", 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0473999999999997e-05 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.300 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241908 5.b 5:d4d086d0:::benchmark_data_smithi032_140417_object108517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.717789+0000", 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.466655664000001, 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.88166062499999998, 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241908 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.301 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717789+0000", 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717789+0000", 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717791+0000", 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5939999999999999e-06 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.302 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717795+0000", 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8850000000000001e-06 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717806+0000", 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0655999999999999e-05 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717813+0000", 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3189999999999997e-06 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.303 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717870+0000", 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6643e-05 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717876+0000", 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9780000000000002e-06 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570288+0000", 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.85241283000000001 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570403+0000", 2024-09-21T10:09:53.304 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114885 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570518+0000", 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114623 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579324+0000", 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0088064619999999993 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.589427+0000", 2024-09-21T10:09:53.305 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010103031 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599354+0000", 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0099262310000000006 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599411+0000", 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7117999999999998e-05 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.306 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599450+0000", 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9369999999999997e-05 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241909 5.b 5:d4aa5536:::benchmark_data_smithi032_140417_object108518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.722478+0000", 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.461966799000002, 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.87717801399999995, 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.307 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241909 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722478+0000", 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722478+0000", 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.308 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722480+0000", 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.172e-06 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722485+0000", 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1379999999999998e-06 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722495+0000", 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0641e-05 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.309 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722503+0000", 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6070000000000002e-06 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722559+0000", 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6514999999999997e-05 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722568+0000", 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9560000000000003e-06 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.310 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.574779+0000", 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.85221065900000004 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.574885+0000", 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010624300000000001 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.575000+0000", 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114509 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.311 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579378+0000", 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0043779769999999999 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.589496+0000", 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010117917000000001 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599561+0000", 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010065362 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.312 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599615+0000", 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4311999999999998e-05 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599656+0000", 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1006e-05 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.313 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241965 5.b 5:deabbc9a:::benchmark_data_smithi032_140417_object108574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.798408+0000", 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 37.386036738000001, 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.80150054400000004, 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241965 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.314 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.798408+0000", 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.798408+0000", 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.798409+0000", 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7000000000000004e-07 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.315 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.798412+0000", 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.072e-06 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.798421+0000", 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5529999999999993e-06 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.798427+0000", 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1959999999999996e-06 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.316 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.798498+0000", 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1598000000000001e-05 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.798509+0000", 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0162e-05 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.584217+0000", 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.78570832300000004 2024-09-21T10:09:53.317 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.584363+0000", 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014587700000000001 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.584471+0000", 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000107905 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.589536+0000", 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0050650460000000001 2024-09-21T10:09:53.318 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599083+0000", 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0095469519999999992 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599828+0000", 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00074487800000000003 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599871+0000", 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3019999999999998e-05 2024-09-21T10:09:53.319 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599909+0000", 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8291999999999999e-05 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:09:53.320 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:09:53.320 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:53.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:53.449 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:53.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:53.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:53.577 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:53.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:53.705 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:53.731 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:53.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:53.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:53 smithi045 ceph-mon[95976]: pgmap v510: 99 pgs: 99 active+clean; 136 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 86 op/s 2024-09-21T10:09:53.787 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:53 smithi032 ceph-mon[101299]: pgmap v510: 99 pgs: 99 active+clean; 136 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 86 op/s 2024-09-21T10:09:53.787 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:53 smithi032 ceph-mon[105748]: pgmap v510: 99 pgs: 99 active+clean; 136 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 86 op/s 2024-09-21T10:09:53.832 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:53.859 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:53.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:53.947 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:09:53.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:54.012 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:54.114 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:54.140 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:54.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:54.192 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:09:54.192 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:09:54.192 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:09:54.192 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:09:54.192 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:09:54.192 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:09:54.192 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:09:54.192 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:09:54.197 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:09:54.198 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:09:54.216 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:09:54.216 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:09:54.232 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:09:54.233 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:09:54.242 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:54.249 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:09:54.249 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:09:54.267 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:54.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:54.369 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:54.391 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:54.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:54.492 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:54.519 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:54.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:54.621 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:54.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:54.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:54.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:54.772 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:54.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:54 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1646191585' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:54.874 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:54.901 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:54.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:54.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:54 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1646191585' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:54 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1646191585' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:55.003 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:55.028 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:55.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:55.130 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:55.156 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:55.258 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:55.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:55.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:55.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:55.410 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:55.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:55.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:55.538 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:55.640 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:55.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:55.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:55.676 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:55 smithi032 ceph-mon[101299]: pgmap v511: 99 pgs: 99 active+clean; 82 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 85 op/s 2024-09-21T10:09:55.676 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:55 smithi032 ceph-mon[105748]: pgmap v511: 99 pgs: 99 active+clean; 82 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 85 op/s 2024-09-21T10:09:55.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:55.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:55 smithi045 ceph-mon[95976]: pgmap v511: 99 pgs: 99 active+clean; 82 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 85 op/s 2024-09-21T10:09:55.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:55.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:55.896 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:55.923 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:55.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:56.026 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:56.052 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:56.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:56.089 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:09:56.092 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:56.092 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:56.154 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:56.181 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:56.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:56.282 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:56.308 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:56.410 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:56.432 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:56.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:56.534 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:56.561 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:56.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:56.663 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:56.685 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:56.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:56.787 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:56.810 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:56.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:56.934 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:56.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:57.036 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:57.063 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:57.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:57.164 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:57.191 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:57.293 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:57.319 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:57.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:57.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:09:57.421 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:57.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:57.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:57.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:57.575 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:57.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:57.639 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:57 smithi032 ceph-mon[105748]: pgmap v512: 99 pgs: 99 active+clean; 5.9 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 13 op/s 2024-09-21T10:09:57.639 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:57 smithi032 ceph-mon[101299]: pgmap v512: 99 pgs: 99 active+clean; 5.9 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 13 op/s 2024-09-21T10:09:57.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:57.700 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:57.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:57 smithi045 ceph-mon[95976]: pgmap v512: 99 pgs: 99 active+clean; 5.9 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 13 op/s 2024-09-21T10:09:57.801 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:57.823 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:57.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:57.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:57.951 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:57.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:57.968 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:57.985 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:58.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:58.076 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:58.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:58.177 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:58.203 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:58.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:58.305 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:58.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:58.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:58.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:58.453 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:58.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:58.554 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:58.580 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:58.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:58.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:58.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:58.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:58.810 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:58.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.835 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:58.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:58.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:58.959 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:58.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:59.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:59.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.089 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:59.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:59.190 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:59.213 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:59.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:59.314 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:59.336 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:59.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:59.438 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:59.463 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:59.501 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_0' removed 2024-09-21T10:09:59.564 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:59.589 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:59.691 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:59.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:59 smithi032 ceph-mon[101299]: pgmap v513: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:09:59.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1353217025' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:09:59.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:09:59 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:09:59.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:59 smithi032 ceph-mon[105748]: pgmap v513: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:09:59.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1353217025' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:09:59.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:09:59 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:09:59.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:09:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:09:59] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.51.0" 2024-09-21T10:09:59.715 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:59.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:59 smithi045 ceph-mon[95976]: pgmap v513: 99 pgs: 99 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:09:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1353217025' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:09:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:09:59 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:09:59.816 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:59.840 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:09:59.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:59.942 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:59.965 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:09:59.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:00.067 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:00.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:00.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:00.119 INFO:teuthology.orchestra.run.smithi032.stdout:moved 0 / 219 (0%) 2024-09-21T10:10:00.119 INFO:teuthology.orchestra.run.smithi032.stdout:avg 27.375 2024-09-21T10:10:00.119 INFO:teuthology.orchestra.run.smithi032.stdout:stddev 9.88607 -> 9.88607 (expected baseline 4.89419) 2024-09-21T10:10:00.120 INFO:teuthology.orchestra.run.smithi032.stdout:min osd.5 with 11 -> 11 pgs (0.401826 -> 0.401826 * mean) 2024-09-21T10:10:00.120 INFO:teuthology.orchestra.run.smithi032.stdout:max osd.1 with 40 -> 40 pgs (1.46119 -> 1.46119 * mean) 2024-09-21T10:10:00.120 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:10:00.120 INFO:teuthology.orchestra.run.smithi032.stdout:oload 1000 2024-09-21T10:10:00.120 INFO:teuthology.orchestra.run.smithi032.stdout:max_change 3 2024-09-21T10:10:00.120 INFO:teuthology.orchestra.run.smithi032.stdout:max_change_osds 4 2024-09-21T10:10:00.120 INFO:teuthology.orchestra.run.smithi032.stdout:average_utilization 0.0020 2024-09-21T10:10:00.120 INFO:teuthology.orchestra.run.smithi032.stdout:overload_utilization 0.0196 2024-09-21T10:10:00.120 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 weight 0.8385 -> 0.8663 2024-09-21T10:10:00.121 INFO:teuthology.orchestra.run.smithi032.stderr:no change 2024-09-21T10:10:00.212 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:00.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.234 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:00.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:00.335 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:00.357 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:00.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:00.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:10:00 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T10:10:00.116+0000 7fce75a2e640 -1 mgr.server reply reply (1) Operation not permitted no change 2024-09-21T10:10:00.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:00.482 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:00.583 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:00.607 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:00.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:00.708 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:00.731 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:00.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:00.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:10:00.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:00 smithi045 ceph-mon[95976]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T10:10:00.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:00 smithi045 ceph-mon[95976]: overall HEALTH_OK 2024-09-21T10:10:00.833 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:00.856 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:00.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:10:00.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:00 smithi032 ceph-mon[101299]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T10:10:00.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:00 smithi032 ceph-mon[101299]: overall HEALTH_OK 2024-09-21T10:10:00.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:10:00.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:00 smithi032 ceph-mon[105748]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T10:10:00.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:00 smithi032 ceph-mon[105748]: overall HEALTH_OK 2024-09-21T10:10:00.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:00.980 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:00.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:01.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:01.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:01.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:01.211 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:01.236 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:01.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:01.338 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:01.363 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:01.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:01.464 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:01.491 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:01.594 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:01.620 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:01.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:01.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:01 smithi032 ceph-mon[101299]: from='client.34896 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:10:01.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:01 smithi032 ceph-mon[101299]: pgmap v515: 73 pgs: 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:01.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:01 smithi032 ceph-mon[105748]: from='client.34896 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:10:01.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:01 smithi032 ceph-mon[105748]: pgmap v515: 73 pgs: 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:01.722 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:01.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:01.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:01 smithi045 ceph-mon[95976]: from='client.34896 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:10:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:01 smithi045 ceph-mon[95976]: pgmap v515: 73 pgs: 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:01.851 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:01.879 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:01.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:01.980 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:02.006 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:02.108 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:02.132 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:02.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:02.234 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:02.261 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:02.363 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:02.386 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:02.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:02.488 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:02.515 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:02.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:02.645 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:02.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:02.747 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:02.773 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:02.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:02.875 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:02.887 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:10:02.887 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:10:02.888 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-09-21T10:10:02.888 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_1 16 2024-09-21T10:10:02.902 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:03.004 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:03.030 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:03.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:03.132 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:03.155 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:03.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:03.256 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:03.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:03.384 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:03.407 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:03.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:03.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:03.536 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:03.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:03.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:03.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:03.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:03.763 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:03 smithi045 ceph-mon[95976]: pgmap v516: 73 pgs: 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:03.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.786 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:03.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:03.808 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:03 smithi032 ceph-mon[105748]: pgmap v516: 73 pgs: 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:03.808 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:03.808 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:03 smithi032 ceph-mon[101299]: pgmap v516: 73 pgs: 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:03.808 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:03.887 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:03.913 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:03.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:04.015 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:04.037 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:04.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:04.139 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:04.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:04.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:04.262 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:04.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:04.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:04.387 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:04.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:04.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:04.471 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:10:04.472 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:10:04.479 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:04.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:04.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:04.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:04.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:04.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:04.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:04.684 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:04.763 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.786 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:04.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:04.887 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:04.911 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:04.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:05.012 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:05.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:05.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:05.163 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:05.265 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:05.292 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:05.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:05.393 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:05.420 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:05.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:05.522 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:05.548 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:05.650 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:05.673 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:05.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:05.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:05 smithi032 ceph-mon[101299]: pgmap v517: 73 pgs: 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:05 smithi032 ceph-mon[105748]: pgmap v517: 73 pgs: 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:05.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:05 smithi045 ceph-mon[95976]: pgmap v517: 73 pgs: 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:05.797 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:05.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:05.899 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:05.921 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:05.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:06.023 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:06.045 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:06.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:06.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:06.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.174 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:06.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:06.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:06.298 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:06.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:06.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:06.423 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:06.516 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:10:06.524 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:06.549 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:06.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:06.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:06.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2569854538' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:10:06.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:06 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:10:06.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2569854538' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:10:06.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:06 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:10:06.781 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:06.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2569854538' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:10:06.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:06 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:06.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:06.909 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:06.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.932 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:06.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:07.033 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:07.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.060 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:07.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:07.161 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:07.185 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:07.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:07.287 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:07.313 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:07.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:07.415 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:07.441 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:07.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:07.533 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_1' created 2024-09-21T10:10:07.543 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:07.568 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:07.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:07.669 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:07.692 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:07 smithi045 ceph-mon[95976]: pgmap v518: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:07 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:10:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:07 smithi045 ceph-mon[95976]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T10:10:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:07 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2898981499' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T10:10:07.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T10:10:07.794 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:07.820 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:07.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[101299]: pgmap v518: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[101299]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2898981499' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[105748]: pgmap v518: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:10:07.872 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[105748]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T10:10:07.873 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2898981499' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T10:10:07.873 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T10:10:07.922 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:07.948 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:07.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:08.050 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:08.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:08.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:08.174 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:08.195 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:08.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:08.297 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:08.319 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:08.404 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:10:08.421 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:08.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:08.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:08.570 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:08.592 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:08.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:08.666 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:08 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-21T10:10:08.667 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:08 smithi032 ceph-mon[105748]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T10:10:08.667 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:08 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-21T10:10:08.667 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:08 smithi032 ceph-mon[101299]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T10:10:08.694 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:08.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:08.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:08.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:08 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-21T10:10:08.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:08 smithi045 ceph-mon[95976]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T10:10:08.823 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:08.846 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:08.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:08.948 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:08.970 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:10:08.970 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:10:08.970 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:10:08.970 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd out 1 2024-09-21T10:10:08.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.971 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:08.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:09.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:09.100 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:09.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:09.201 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:09.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:09.325 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:09.352 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:09.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:09.453 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:09.476 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:09.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:09.577 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:09.604 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:09.706 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:09.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:09 smithi032 ceph-mon[101299]: pgmap v521: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:09.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:09 smithi032 ceph-mon[101299]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:10:09.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:09 smithi032 ceph-mon[101299]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T10:10:09.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:10:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:10:09] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.51.0" 2024-09-21T10:10:09.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:09 smithi032 ceph-mon[105748]: pgmap v521: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:09.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:09 smithi032 ceph-mon[105748]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:10:09.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:09 smithi032 ceph-mon[105748]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T10:10:09.730 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:09.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:09.745 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-09-21T10:10:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:09 smithi045 ceph-mon[95976]: pgmap v521: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:09 smithi045 ceph-mon[95976]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:10:09.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:09 smithi045 ceph-mon[95976]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T10:10:09.832 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:09.859 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:09.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:09.961 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:09.985 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:09.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:10.087 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:10.112 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:10.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:10.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:10.238 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:10.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:10.263 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:10.297 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:10.340 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:10.364 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:10.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:10.466 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:10.489 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:10.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:10.591 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:10.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.618 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:10.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:10.718 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:10.744 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:10.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:10.756 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:10 smithi032 ceph-mon[101299]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T10:10:10.756 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:10 smithi032 ceph-mon[105748]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T10:10:10.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:10 smithi045 ceph-mon[95976]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T10:10:10.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:10.875 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:10.976 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:11.003 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:11.104 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:11.127 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:11.210 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:10:11.211 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:11.220 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:10:11.221 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:11.228 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:11.237 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:10:11.237 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:11.253 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:10:11.253 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:11.255 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:11.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:11.270 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:10:11.270 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:11.287 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:10:11.287 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:11.303 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:10:11.303 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:11.320 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:10:11.320 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:11.356 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:11.383 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:11.484 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:11.507 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:11.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:11.608 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:11.635 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:11.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:11.688 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:11 smithi032 ceph-mon[105748]: pgmap v524: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:11.688 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:11 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/281094496' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:11.689 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:11 smithi032 ceph-mon[101299]: pgmap v524: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:11.689 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:11 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/281094496' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:11.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:11.758 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:11.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:11.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:11 smithi045 ceph-mon[95976]: pgmap v524: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:11.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:11 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/281094496' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:11.861 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:11.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:11.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:11.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:12.016 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:12.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:12.117 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:12.144 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:12.239 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:10:12.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:12.272 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:12.374 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:12.400 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:12.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:12.502 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:12.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:12.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:12.627 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:12.654 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:12.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:12.756 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:12.779 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:12.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:12.881 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:12.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.905 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:12.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:13.006 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:13.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.034 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:13.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:13.135 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:13.158 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:13.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:13.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:13.287 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:13.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:13.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:13.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:13.490 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:13.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:13.534 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:13.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:13.636 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:13.662 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:13.764 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:13.790 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:13.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:13.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:13.915 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:13 smithi032 ceph-mon[101299]: pgmap v525: 89 pgs: 1 creating+peering, 88 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:13 smithi032 ceph-mon[105748]: pgmap v525: 89 pgs: 1 creating+peering, 88 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:14.018 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:13 smithi045 ceph-mon[95976]: pgmap v525: 89 pgs: 1 creating+peering, 88 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:14.042 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:14.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:14.144 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:14.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:14.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:14.272 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:14.294 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:14.396 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:14.423 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:14.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:14.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:14.550 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:14.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:14.652 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:14.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:14.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:14.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:14.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:14.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:14.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:14.924 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:14.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:15.026 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:15.048 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:15.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:15.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:15.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:15.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:15.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:15.299 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:15.401 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:15.425 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:15.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:15.547 INFO:teuthology.orchestra.run.smithi032.stderr:marked out osd.1. 2024-09-21T10:10:15.582 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:15.608 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:15.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:15.659 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:15 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:10:15.499+0000 7f532c355640 -1 mon.a@0(leader).osd e195 definitely_dead 0 2024-09-21T10:10:15.659 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:15 smithi032 ceph-mon[101299]: pgmap v526: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:15.659 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:15 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/426408837' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:10:15.659 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:15 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:10:15.659 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:15 smithi032 ceph-mon[101299]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:10:15.659 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:15 smithi032 ceph-mon[105748]: pgmap v526: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:15.660 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:15 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/426408837' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:10:15.660 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:15 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:10:15.660 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:15 smithi032 ceph-mon[105748]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:10:15.660 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:10:15.660 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:10:15.660 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:10:15.660 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:10:15.660 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:10:15.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:15.732 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:15.834 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:15.860 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:15.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:15.962 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:15.985 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:15.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:15 smithi045 ceph-mon[95976]: pgmap v526: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:10:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:15 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/426408837' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:10:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:15 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:10:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:15 smithi045 ceph-mon[95976]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:10:16.046 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:10:16.086 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:16.113 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:16.215 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:16.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:16.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:16.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:16.371 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:16.473 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:16.499 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:16.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:16.601 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:16.624 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:16.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:16.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:16 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:10:16.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:16 smithi032 ceph-mon[101299]: osdmap e196: 8 total, 8 up, 7 in 2024-09-21T10:10:16.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:16 smithi032 ceph-mon[101299]: osdmap e197: 8 total, 8 up, 7 in 2024-09-21T10:10:16.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:16 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:10:16.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:16 smithi032 ceph-mon[105748]: osdmap e196: 8 total, 8 up, 7 in 2024-09-21T10:10:16.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:16 smithi032 ceph-mon[105748]: osdmap e197: 8 total, 8 up, 7 in 2024-09-21T10:10:16.726 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:16.748 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:16.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:16.849 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:16.872 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:16.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:16.973 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:17.000 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:16 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:10:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:16 smithi045 ceph-mon[95976]: osdmap e196: 8 total, 8 up, 7 in 2024-09-21T10:10:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:16 smithi045 ceph-mon[95976]: osdmap e197: 8 total, 8 up, 7 in 2024-09-21T10:10:17.103 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:17.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:17.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:17.230 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:17.255 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:17.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:17.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:17.382 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:17.484 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:17.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:17.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:17.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:17.635 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:17.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:17.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:17.759 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:17.861 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:17.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:17.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:17 smithi032 ceph-mon[101299]: pgmap v528: 89 pgs: 89 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-21T10:10:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:17 smithi032 ceph-mon[101299]: osdmap e198: 8 total, 8 up, 7 in 2024-09-21T10:10:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:17 smithi032 ceph-mon[105748]: pgmap v528: 89 pgs: 89 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-21T10:10:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:17 smithi032 ceph-mon[105748]: osdmap e198: 8 total, 8 up, 7 in 2024-09-21T10:10:17.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:18.009 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:18.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:17 smithi045 ceph-mon[95976]: pgmap v528: 89 pgs: 89 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-21T10:10:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:17 smithi045 ceph-mon[95976]: osdmap e198: 8 total, 8 up, 7 in 2024-09-21T10:10:18.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:10:18.111 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:18.138 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:18.240 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:18.267 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:18.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:18.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:18.399 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:18.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:18.408 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:18.410 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:18.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:18.532 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:18.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:18.633 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:18.660 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:18.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:18.762 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:18.784 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:18.886 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:18.913 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:18.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:19.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:19.037 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:19.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:19.138 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:19.165 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:19.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:19.267 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:19.290 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:19.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:19.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:19.419 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:19.512 INFO:teuthology.orchestra.run.smithi032.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-09-21T10:10:19.520 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:19.545 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:19.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:19.648 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:19.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:19.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:19.736 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:10:19] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-09-21T10:10:19.736 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[105748]: pgmap v531: 89 pgs: 2 peering, 1 unknown, 86 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:19.736 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:19.736 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:19.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[105748]: osdmap e199: 8 total, 8 up, 7 in 2024-09-21T10:10:19.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/982350200' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:10:19.737 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:10:19.737 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[101299]: pgmap v531: 89 pgs: 2 peering, 1 unknown, 86 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:19.737 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:19.737 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:19.737 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[101299]: osdmap e199: 8 total, 8 up, 7 in 2024-09-21T10:10:19.737 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/982350200' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:10:19.737 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:19 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:10:19.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:19 smithi045 ceph-mon[95976]: pgmap v531: 89 pgs: 2 peering, 1 unknown, 86 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:19 smithi045 ceph-mon[95976]: osdmap e199: 8 total, 8 up, 7 in 2024-09-21T10:10:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/982350200' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:10:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:19 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:10:19.799 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:19.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:19.900 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:19.927 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:20.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:20.055 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:20.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:20.157 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:20.183 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:20.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:20.285 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:20.311 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:20.413 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:20.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:20.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:20.541 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:20.568 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:20.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:20.670 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:20.696 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:20.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:20 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:10:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:20 smithi032 ceph-mon[101299]: osdmap e200: 8 total, 8 up, 7 in 2024-09-21T10:10:20.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:20 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:10:20.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:20 smithi032 ceph-mon[105748]: osdmap e200: 8 total, 8 up, 7 in 2024-09-21T10:10:20.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:20 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:10:20.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:20 smithi045 ceph-mon[95976]: osdmap e200: 8 total, 8 up, 7 in 2024-09-21T10:10:20.798 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:20.824 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:20.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:20.926 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:20.953 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:20.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:21.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:21.080 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:21.128 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:21.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.207 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:21.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:21.308 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:21.332 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:21.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:21.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.459 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:21.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:21.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.583 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:21.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:21.684 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:21.706 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:21.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:21 smithi045 ceph-mon[95976]: pgmap v534: 89 pgs: 5 peering, 1 unknown, 83 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 1.1 MiB/s, 11 objects/s recovering 2024-09-21T10:10:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:21 smithi045 ceph-mon[95976]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:10:21.800 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:21 smithi032 ceph-mon[101299]: pgmap v534: 89 pgs: 5 peering, 1 unknown, 83 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 1.1 MiB/s, 11 objects/s recovering 2024-09-21T10:10:21.800 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:21 smithi032 ceph-mon[101299]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:10:21.800 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:21 smithi032 ceph-mon[105748]: pgmap v534: 89 pgs: 5 peering, 1 unknown, 83 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 1.1 MiB/s, 11 objects/s recovering 2024-09-21T10:10:21.800 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:21 smithi032 ceph-mon[105748]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:10:21.808 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:21.831 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:21.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:21.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:21.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:21.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:22.057 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:22.081 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:22.130 DEBUG:teuthology.orchestra.run.smithi032:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-09-21T10:10:22.133 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:10:22.133 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:10:22.184 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:22.186 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:hints = 1 2024-09-21T10:10:22.186 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:10:22.186 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object prefix: benchmark_data_smithi032_171826 2024-09-21T10:10:22.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:10:22.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:10:22.227 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:22.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:22.329 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:22.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.366 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:22.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:22.389 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:10:22.389 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:10:22.389 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-09-21T10:10:22.389 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:10:22.389 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:10:22.389 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-09-21T10:10:22.443 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:10:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=sqlstore.transactions t=2024-09-21T10:10:22.09289471Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T10:10:22.467 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:22.505 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:10:22.505 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:10:22.506 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:10:22.506 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:10:22.506 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:10:22.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:22.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:22.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:22.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:22.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:22.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.507 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:22.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:22.538 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:10:22.538 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:22.577 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:10:22.577 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:22.609 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:22.616 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:10:22.617 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:22.617 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:22 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2968647769' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:22.617 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:22 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2968647769' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:22.649 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:22.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:22.750 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:22.763 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:10:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:22 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2968647769' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:22.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:22.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:22.927 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:22.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:23.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:23.067 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:23.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:23.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:23.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 1 16 3671 3655 228.425 228.438 0.00246191 0.00436761 2024-09-21T10:10:23.210 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:23.313 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:23.350 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:23.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:23.491 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:23.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:23.532 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:10:23.593 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:23.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.631 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:23.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:23.731 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:23.768 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:23.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:23 smithi045 ceph-mon[95976]: pgmap v535: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 813 KiB/s, 10 objects/s recovering 2024-09-21T10:10:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/132200234' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:23.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:23.906 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:23.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:23.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:23 smithi032 ceph-mon[105748]: pgmap v535: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 813 KiB/s, 10 objects/s recovering 2024-09-21T10:10:23.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/132200234' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:23 smithi032 ceph-mon[101299]: pgmap v535: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 813 KiB/s, 10 objects/s recovering 2024-09-21T10:10:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/132200234' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:24.008 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:24.046 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:24.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:24.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:24.182 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:24.186 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 2 16 7394 7378 230.545 232.688 0.00533918 0.00432827 2024-09-21T10:10:24.284 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:24.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.323 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:24.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:24.424 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:24.460 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:24.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:24.561 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:24.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:24.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:24.635 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:24 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:24.635 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:24 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/132200234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:24.635 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:24 smithi045 ceph-mon[95976]: osdmap e201: 8 total, 8 up, 7 in 2024-09-21T10:10:24.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:24 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:24 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/132200234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:24 smithi032 ceph-mon[105748]: osdmap e201: 8 total, 8 up, 7 in 2024-09-21T10:10:24.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:24 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:24.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:24 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/132200234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:24.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:24 smithi032 ceph-mon[101299]: osdmap e201: 8 total, 8 up, 7 in 2024-09-21T10:10:24.744 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:24.817 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:10:24.817 INFO:teuthology.orchestra.run.smithi045.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:10:24.817 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:10:24.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:24.886 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:24.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:24.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:25.029 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:25.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:25.131 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:25.167 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:25.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 3 16 10680 10664 222.149 205.375 0.00517279 0.00449309 2024-09-21T10:10:25.269 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:25.308 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:25.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:25.409 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:25.450 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:25.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:25.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:25.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.594 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:25.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:25.694 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:25.730 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:25.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:25.782 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:25 smithi032 ceph-mon[101299]: pgmap v537: 89 pgs: 3 peering, 86 active+clean; 40 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 5.9 MiB/s wr, 93 op/s; 779 KiB/s, 9 objects/s recovering 2024-09-21T10:10:25.782 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:25 smithi032 ceph-mon[105748]: pgmap v537: 89 pgs: 3 peering, 86 active+clean; 40 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 5.9 MiB/s wr, 93 op/s; 779 KiB/s, 9 objects/s recovering 2024-09-21T10:10:25.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:25.862 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:25 smithi045 ceph-mon[95976]: pgmap v537: 89 pgs: 3 peering, 86 active+clean; 40 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 5.9 MiB/s wr, 93 op/s; 779 KiB/s, 9 objects/s recovering 2024-09-21T10:10:25.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:25.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:25.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:26.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.037 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:26.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:26.097 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:10:26.098 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:10:26.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:10:26.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:26.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.176 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:26.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:26.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 4 16 13645 13629 212.936 185.312 0.00550832 0.00468929 2024-09-21T10:10:26.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:26.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.317 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:26.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:26.418 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.458 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:26.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:26.559 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:26.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:26.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:26.699 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:26.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:26.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:26.840 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:26.877 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:26.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:26.979 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:27.015 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:27.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:27.117 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:27.152 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:27.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:27.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 5 16 16573 16557 206.945 183 0.00561998 0.00482556 2024-09-21T10:10:27.254 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:27.295 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:10:27.295 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:27.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:27.391 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:27.397 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:27.435 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:27.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:27.536 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:10:27.537 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:27.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.589 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:27.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:27.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:27 smithi045 ceph-mon[95976]: pgmap v538: 89 pgs: 89 active+clean; 322 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 41 MiB/s wr, 655 op/s; 603 KiB/s, 7 objects/s recovering 2024-09-21T10:10:27.670 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:10:27.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:27.732 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:27.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:27.834 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:27.873 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:27.874 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:27 smithi032 ceph-mon[101299]: pgmap v538: 89 pgs: 89 active+clean; 322 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 41 MiB/s wr, 655 op/s; 603 KiB/s, 7 objects/s recovering 2024-09-21T10:10:27.874 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:27 smithi032 ceph-mon[105748]: pgmap v538: 89 pgs: 89 active+clean; 322 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 41 MiB/s wr, 655 op/s; 603 KiB/s, 7 objects/s recovering 2024-09-21T10:10:27.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:27.976 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:28.014 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:28.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:28.115 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:28.150 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:28.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:28.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 6 16 19263 19247 200.473 168.125 0.00308333 0.00498163 2024-09-21T10:10:28.252 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:28.288 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:28.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:28.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:28.431 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:28.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:28.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:28.574 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:28.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:28.676 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:28.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:28.816 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:28.855 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:28.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:28.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:28.995 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:28.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:29.097 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:29.143 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:29.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:29.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 7 16 22098 22082 197.138 177.188 0.00890536 0.00506663 2024-09-21T10:10:29.244 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:29.287 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:29.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:29.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:29.430 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:29.532 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:29.559 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:29.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:29.619 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:29 smithi032 ceph-mon[101299]: pgmap v539: 89 pgs: 89 active+clean; 619 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 2 B/s, 2 objects/s recovering 2024-09-21T10:10:29.620 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:10:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:10:29] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-09-21T10:10:29.620 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:29 smithi032 ceph-mon[105748]: pgmap v539: 89 pgs: 89 active+clean; 619 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 2 B/s, 2 objects/s recovering 2024-09-21T10:10:29.661 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:29.703 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:29.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:29.805 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:29.846 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:29.847 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:29 smithi045 ceph-mon[95976]: pgmap v539: 89 pgs: 89 active+clean; 619 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 2 B/s, 2 objects/s recovering 2024-09-21T10:10:29.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:29.948 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:29.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.986 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:29.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:30.087 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:10:30.087 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:10:30.087 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:10:30.087 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:10:30.087 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:10:30.087 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:10:30.087 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:10:30.087 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:10:30.087 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:30.088 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:30.088 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:30.088 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:30.088 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:30.088 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:30.088 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:30.088 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:30.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:30.090 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:10:30.090 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:10:30.127 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:30.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 8 16 24250 24234 189.307 134.5 0.0125736 0.00527483 2024-09-21T10:10:30.229 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:30.265 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:30.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:30.367 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:30.405 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:30.506 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:30.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:30.610 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:10:30.610 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:10:30.611 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-09-21T10:10:30.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.0 reset_purged_snaps_last 2024-09-21T10:10:30.639 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:30.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:30.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:30.775 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:30.809 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:30.863 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:30 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2956472981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:30.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:30 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2956472981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:30.894 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:30.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:30.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:30.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:30 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2956472981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:31.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:31.083 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:31.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:31.185 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:31.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 9 16 26146 26130 181.439 118.5 0.00639984 0.00550515 2024-09-21T10:10:31.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.219 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:31.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:31.320 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:31.369 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:31.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:31.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:31.507 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:31.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:31.608 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:31.647 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:31.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:31.665 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:31.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:31.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:31 smithi045 ceph-mon[95976]: pgmap v540: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:31.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:31.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:31.831 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:10:31.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:31.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:31.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:31 smithi032 ceph-mon[105748]: pgmap v540: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:31.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:31 smithi032 ceph-mon[101299]: pgmap v540: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:32.035 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.089 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:32.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:32.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 10 16 28441 28425 177.629 143.438 0.00406105 0.00562426 2024-09-21T10:10:32.190 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:32.216 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:10:32.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:32.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:32.324 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:32.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:32.451 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:32.489 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:32.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.629 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:32.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:32.639 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:32 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:10:32.729 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:32.767 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:32.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:32.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:32.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:32.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:32.909 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:10:32.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:32 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:10:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:32 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:10:33.006 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:33.055 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:33.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:33.157 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:33.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 11 16 30544 30528 173.428 131.438 0.00593011 0.00576034 2024-09-21T10:10:33.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:33.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:33.296 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:33.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:33.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:33.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:33.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:33.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:33.573 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:33.623 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:33.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:33.733 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:33 smithi045 ceph-mon[95976]: pgmap v541: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-21T10:10:33.733 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:33.734 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:33.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:33.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:33.855 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:33 smithi032 ceph-mon[101299]: pgmap v541: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-21T10:10:33.856 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:33.856 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:33.856 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:33 smithi032 ceph-mon[105748]: pgmap v541: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-21T10:10:33.856 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:33.856 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:33.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:33.903 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:33.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:34.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:34.039 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:34.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:34.141 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:34.183 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:34.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 12 16 32480 32464 169.058 121 0.0109422 0.00590964 2024-09-21T10:10:34.245 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:10:34.245 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:10:34.245 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:10:34.245 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:10:34.245 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:10:34.285 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:34.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:34.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:34.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:34.460 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:34.562 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:34.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:34.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:34.707 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:34.743 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:34.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:34.845 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:34.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.880 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:34.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:34.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:35.016 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:35.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:35.118 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:35.165 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:35.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 13 16 34480 34464 165.668 125 0.0186434 0.00602828 2024-09-21T10:10:35.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:35.303 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:35.405 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:35.428 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:35.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:35.530 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:35.568 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:35.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:35.670 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:35.692 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:35.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:35.794 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:35.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.827 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:35.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:35.928 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:35.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:35.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:35 smithi032 ceph-mon[101299]: pgmap v542: 89 pgs: 89 active+clean; 1.5 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-21T10:10:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:35 smithi032 ceph-mon[105748]: pgmap v542: 89 pgs: 89 active+clean; 1.5 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-21T10:10:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:35 smithi045 ceph-mon[95976]: pgmap v542: 89 pgs: 89 active+clean; 1.5 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-21T10:10:36.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:36.088 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:36.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:36.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 14 16 36091 36075 161.026 100.688 0.033275 0.00620131 2024-09-21T10:10:36.190 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:36.216 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:36.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:36.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:36.350 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:36.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:36.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:36.474 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:36.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:36.575 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:36.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.613 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:36.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:36.713 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:36.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:36.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:36.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:36.881 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:36.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:36.983 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:37.021 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:37.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:37.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:37.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.154 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:37.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:37.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 15 16 37635 37619 156.724 96.5 0.00191459 0.00635429 2024-09-21T10:10:37.255 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:37.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.288 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:37.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:37.389 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:37.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.435 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:37.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:37.536 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:37.558 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:37.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:37.660 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:37.694 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:37.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:37.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:37.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:37.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:37 smithi032 ceph-mon[105748]: pgmap v543: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 158 MiB/s wr, 2.52k op/s 2024-09-21T10:10:37.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:37 smithi032 ceph-mon[101299]: pgmap v543: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 158 MiB/s wr, 2.52k op/s 2024-09-21T10:10:37.970 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:37.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:37 smithi045 ceph-mon[95976]: pgmap v543: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 158 MiB/s wr, 2.52k op/s 2024-09-21T10:10:38.071 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:38.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:38.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:10:38.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 16 16 39222 39206 153.128 99.1875 0.00879442 0.00652476 2024-09-21T10:10:38.212 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:38.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:38.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:38.361 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:38.388 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:38.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:38.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:38.513 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:38.614 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:38.639 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:38.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:38.740 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:38.762 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:38.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:38.864 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:38.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.898 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:38.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:38.999 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:39.042 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:39.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:39.144 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:39.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.171 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:39.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:39.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 17 16 40373 40357 148.352 71.9375 0.00226945 0.00672436 2024-09-21T10:10:39.272 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:39.299 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:39.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:39.401 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:39.437 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:39.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:39.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:39.566 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:39.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:39.630 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:10:39.669 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:39.708 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:39.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:39.809 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:39.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.848 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:39.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:39.949 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:39 smithi032 ceph-mon[101299]: pgmap v544: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-09-21T10:10:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/477022392' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:39 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:39 smithi032 ceph-mon[105748]: pgmap v544: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-09-21T10:10:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/477022392' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:39 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:39.958 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:10:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:10:39] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.51.0" 2024-09-21T10:10:39.980 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:39.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:39 smithi045 ceph-mon[95976]: pgmap v544: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-09-21T10:10:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:39 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/477022392' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:39 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:40.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:40.120 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:40.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 18 16 41793 41777 145.04 88.75 0.00212476 0.00688333 2024-09-21T10:10:40.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:40.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:40.361 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:40.395 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:40.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:40.469 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.2 reset_purged_snaps_last 2024-09-21T10:10:40.497 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:40.533 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:40.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:40.635 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.678 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:40.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:40.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:40 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:40.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:40 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:10:40.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:40 smithi032 ceph-mon[101299]: osdmap e202: 8 total, 8 up, 7 in 2024-09-21T10:10:40.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:40 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:40.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:40 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:10:40.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:40 smithi032 ceph-mon[105748]: osdmap e202: 8 total, 8 up, 7 in 2024-09-21T10:10:40.756 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:40.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:40.801 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:40.903 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:40.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.944 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:40.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:41.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:40 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:41.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:40 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:10:41.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:40 smithi045 ceph-mon[95976]: osdmap e202: 8 total, 8 up, 7 in 2024-09-21T10:10:41.046 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:41.090 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:41.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 19 16 43360 43344 142.559 97.9375 0.0159155 0.00700969 2024-09-21T10:10:41.192 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:41.241 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:41.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:41.374 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:41.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:41.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:41.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:41.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:41.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:41.742 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:41.781 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:41.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:41.883 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:41.921 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:41.923 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:41 smithi032 ceph-mon[101299]: pgmap v546: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-21T10:10:41.924 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:41 smithi032 ceph-mon[105748]: pgmap v546: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-21T10:10:42.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:42.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:41 smithi045 ceph-mon[95976]: pgmap v546: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-21T10:10:42.045 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:42.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:42.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:42.183 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:42.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:42.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:10:42.188323+0000 min lat: 0.00124686 max lat: 0.234546 avg lat: 0.00712208 2024-09-21T10:10:42.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:10:42.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 20 16 44924 44908 140.318 97.75 0.0106419 0.00712208 2024-09-21T10:10:42.285 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:42.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.309 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:42.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:42.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:42.449 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:42.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:42.550 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:42.589 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:42.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:42.691 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:42.731 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:42.833 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:42.844 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:42.859 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:10:42.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:42.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:42.989 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:43.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:43.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:43.127 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.154 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:43.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:43.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 21 16 46570 46554 138.535 102.875 0.00318519 0.00721354 2024-09-21T10:10:43.255 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:43.292 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:43.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:43.394 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:43.439 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:43.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:43.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:43.577 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:43.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:43.647 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:10:43.647 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:43.679 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:43.680 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:10:43.680 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:43.704 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:10:43.704 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.726 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:43.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:43.727 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:10:43.727 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:43.756 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:10:43.756 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:43.787 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:10:43.788 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:43.821 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:10:43.822 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:43.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:43.858 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:10:43.858 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:43.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.867 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:43.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:43.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:43 smithi032 ceph-mon[101299]: pgmap v547: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-21T10:10:43.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:43 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/441547213' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:43.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:43 smithi032 ceph-mon[105748]: pgmap v547: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-21T10:10:43.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:43 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/441547213' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:43.969 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:44.003 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:43 smithi045 ceph-mon[95976]: pgmap v547: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-21T10:10:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:43 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/441547213' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:44.105 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:44.143 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:44.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 22 16 48275 48259 137.081 106.562 0.0144813 0.00728943 2024-09-21T10:10:44.245 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:44.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:44.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:44.384 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:44.436 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:44.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:44.538 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:44.576 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:44.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:44.678 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:44.713 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:44.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:44.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:10:44.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:44.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:44.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:45.035 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:45.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:45.085 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:10:45.086 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:10:45.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:45.172 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:45.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:45.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 23 16 49775 49759 135.197 93.75 0.00389365 0.00739133 2024-09-21T10:10:45.274 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:45.307 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:45.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:45.409 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.448 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:45.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:45.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:45.588 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:45.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:45.683 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:45 smithi045 ceph-mon[95976]: pgmap v548: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-21T10:10:45.690 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:45.735 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:45.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:45.806 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:45 smithi032 ceph-mon[101299]: pgmap v548: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-21T10:10:45.807 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:45 smithi032 ceph-mon[105748]: pgmap v548: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-21T10:10:45.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:45.877 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:45.980 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:46.018 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:46.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:46.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.160 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:46.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:46.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 24 16 51504 51488 134.066 108.062 0.0103752 0.00745395 2024-09-21T10:10:46.261 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:46.299 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:46.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:46.401 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:46.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.440 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:46.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:46.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:10:46.541 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:46.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:46.604 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:46.700 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:46.745 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:46.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:10:46.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:46.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:46.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:46.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:47.037 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:10:47.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.045 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:47.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:47.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:47.176 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:47.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:47.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 25 16 52990 52974 132.418 92.875 0.00485681 0.00754691 2024-09-21T10:10:47.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:47.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:47.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:47.422 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:47.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:47.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:47.557 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:47.599 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:47.701 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:47.732 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:47.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:47.801 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:47 smithi045 ceph-mon[95976]: pgmap v549: 89 pgs: 89 active+clean; 2.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-21T10:10:47.834 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:47.848 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:47 smithi032 ceph-mon[105748]: pgmap v549: 89 pgs: 89 active+clean; 2.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-21T10:10:47.849 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:47 smithi032 ceph-mon[101299]: pgmap v549: 89 pgs: 89 active+clean; 2.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-21T10:10:47.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:47.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:47.890 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.3 reset_purged_snaps_last 2024-09-21T10:10:47.975 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:48.013 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:48.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:48.151 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:48.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:48.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 26 16 54698 54682 131.43 106.75 0.0118215 0.00760303 2024-09-21T10:10:48.253 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:48.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.289 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:48.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:48.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:48.433 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:48.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:48.534 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:48.570 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:48.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:48.672 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:48.710 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:48.812 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:48.854 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:48.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:48.865 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:48.956 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:48.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:48.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:49.004 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:49.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:49.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:49.140 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:49.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:49.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 27 16 56312 56296 130.298 100.875 0.00303433 0.00765535 2024-09-21T10:10:49.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:49.279 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:49.381 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:49.417 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:49.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:49.519 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.557 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:49.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:49.658 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:49.696 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:49.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:49.797 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:49.835 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:49.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:49.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:49 smithi032 ceph-mon[101299]: pgmap v550: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-21T10:10:49.844 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:10:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:10:49] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.51.0" 2024-09-21T10:10:49.844 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:49 smithi032 ceph-mon[105748]: pgmap v550: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-21T10:10:49.937 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:49.948 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:49.960 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:49 smithi045 ceph-mon[95976]: pgmap v550: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-21T10:10:50.013 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:10:50.013 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:10:50.024 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:50.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:50.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:50.165 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:50.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 28 16 57840 57824 129.055 95.5 0.0120783 0.00774361 2024-09-21T10:10:50.267 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.306 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:50.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:50.407 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:50.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.446 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:50.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:50.547 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:50.587 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:50.689 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:50.725 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:50.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:50.767 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:10:50.828 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:50.902 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:50.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:51.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:51.044 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:51.129 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:51.129 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:51.129 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:51 smithi032 ceph-mon[105748]: pgmap v551: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-21T10:10:51.130 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:51.130 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:51.130 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:51 smithi032 ceph-mon[101299]: pgmap v551: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-21T10:10:51.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:51.186 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:10:51.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 29 16 59250 59234 127.643 88.125 0.0124602 0.00782848 2024-09-21T10:10:51.196 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:51.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:51.298 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:51.327 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:51.327 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:51.327 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:51 smithi045 ceph-mon[95976]: pgmap v551: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-21T10:10:51.333 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:51.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:51.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:51.470 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:51.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:51.572 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:51.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.610 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:51.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:51.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:51.748 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:51.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:51.871 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:51.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:51.973 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:52.008 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:52.071 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:10:52.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:52.143 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:52.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:52.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 30 16 60819 60803 126.657 98.0625 0.00582293 0.00789001 2024-09-21T10:10:52.245 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:52.285 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:52.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:52.381 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:52 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1585879568' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.381 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:52 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:52.424 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:52.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:52 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1585879568' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:52 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:52 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1585879568' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:52 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.525 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:52.576 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:52.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:52.678 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:52.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.712 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:52.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:52.813 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:52.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.859 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:52.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:52.960 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:52.982 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:52.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:53.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:53.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.122 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:53.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:53.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 31 16 62237 62221 125.43 88.625 0.00606974 0.00796787 2024-09-21T10:10:53.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:53.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:53.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:53.360 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:53.390 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:53.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:53.418 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:10:53.418 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:10:53.418 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:53.418 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[101299]: osdmap e203: 8 total, 8 up, 7 in 2024-09-21T10:10:53.419 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[101299]: pgmap v553: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:10:53.419 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:10:53.419 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:10:53.419 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:53.419 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[105748]: osdmap e203: 8 total, 8 up, 7 in 2024-09-21T10:10:53.419 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:53 smithi032 ceph-mon[105748]: pgmap v553: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:10:53.493 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:53.523 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:53 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:10:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:53 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:10:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:53 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:53 smithi045 ceph-mon[95976]: osdmap e203: 8 total, 8 up, 7 in 2024-09-21T10:10:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:53 smithi045 ceph-mon[95976]: pgmap v553: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:10:53.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:53.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.654 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:53.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:53.755 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:53.773 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:53.791 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:53.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:53.893 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:53.939 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:53.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:54.041 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:54.088 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:54.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:54.190 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:54.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 32 16 63588 63572 124.149 84.4375 0.00253995 0.00804801 2024-09-21T10:10:54.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.221 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:54.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:54.323 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:54.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:54.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.508 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:54.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:54.609 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:54.652 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:54.754 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:54.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.799 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:54.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:54.900 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:54.926 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:54.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:55.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:55.066 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:55.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:55.168 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:55.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 33 16 64727 64711 122.544 71.1875 0.00800505 0.00815477 2024-09-21T10:10:55.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:55.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/445714079' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:55.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:54 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/445714079' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:55.264 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:55.264 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:55.265 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:10:55.265 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:10:55.265 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:10:55.265 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:54 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/445714079' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:55.300 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:55.346 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:55.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:55.448 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:55.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:55.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:55.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:55.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:55.724 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:55.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.748 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:55.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:55.849 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:55.883 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:55.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:55.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:56.022 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:56.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:56.125 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:56.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:56.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:56.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 34 15 65979 65964 121.243 78.3125 0.00582742 0.00824308 2024-09-21T10:10:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:55 smithi032 ceph-mon[101299]: pgmap v554: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:10:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:55 smithi032 ceph-mon[105748]: pgmap v554: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:10:56.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:56.266 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:10:56.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:56.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:55 smithi045 ceph-mon[95976]: pgmap v554: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:10:56.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:56.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:56.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.429 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:56.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:56.530 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:56.567 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:56.669 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:56.703 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:56.769 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:10:56.770 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:10:56.770 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:10:56.770 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:10:56.770 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:10:56.773 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:10:56.773 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:10:56.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:10:56.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:10:56.774 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:56.774 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:56.774 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:56.774 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:56.774 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:56.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:10:56.775 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:10:56.775 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:10:56.775 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:10:56.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:56.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:56.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:56.804 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:56.843 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:56.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:56.945 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:56.981 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:56.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:57.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:57.125 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:57.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:57.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 35 16 67370 67354 120.261 86.875 0.0143564 0.00831048 2024-09-21T10:10:57.226 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:57.262 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:57.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:57.364 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:57.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.402 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:57.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:57.503 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:57.551 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:10:57.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.553 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:57.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:57.654 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:57.696 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:57.798 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:57.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:57.934 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:57 smithi032 ceph-mon[101299]: pgmap v555: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:10:57.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:57 smithi032 ceph-mon[105748]: pgmap v555: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:10:57.965 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:10:57.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:58.077 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:58.102 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:57 smithi045 ceph-mon[95976]: pgmap v555: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:10:58.123 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:58.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:58.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 36 16 68789 68773 119.383 88.6875 0.0114482 0.00837151 2024-09-21T10:10:58.227 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:58.264 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:58.366 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:58.410 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:58.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:58.512 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:58.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.557 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:58.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:58.658 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:58.694 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:58.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:58.796 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:58.836 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:58.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:58.938 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:58.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:59.076 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:59.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.114 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:59.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:59.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 37 16 70171 70155 118.491 86.375 0.00945738 0.00843416 2024-09-21T10:10:59.215 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:59.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.267 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:59.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:59.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:59.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:59.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:59.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:59.541 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:10:59.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:59.642 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:59.677 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:59.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:59.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:59.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.824 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:10:59.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:59.832 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:10:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:10:59] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.51.0" 2024-09-21T10:10:59.925 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233862 5.16 5:6a8e88e5:::benchmark_data_smithi032_140417_object100471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.058459+0000", 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.883365477, 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0285780770000001, 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:59.948 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233862 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058459+0000", 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058459+0000", 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.949 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058457+0000", 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058464+0000", 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.109e-06 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058480+0000", 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5707000000000001e-05 2024-09-21T10:10:59.950 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058488+0000", 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7819999999999994e-06 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058917+0000", 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00042962099999999999 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058929+0000", 2024-09-21T10:10:59.951 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1644999999999999e-05 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965364+0000", 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.90643523400000003 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965388+0000", 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3691999999999999e-05 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.080586+0000", 2024-09-21T10:10:59.952 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.115198087 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.080700+0000", 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011441400000000001 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.081109+0000", 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000408293 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083462+0000", 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0023531110000000002 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083470+0000", 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9209999999999992e-06 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086650+0000", 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031807570000000002 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.954 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086871+0000", 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000221015 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086935+0000", 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3935000000000004e-05 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086969+0000", 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3460000000000002e-05 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:10:59.955 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087037+0000", 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7974999999999999e-05 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233884 5.16 5:6f45c8b4:::benchmark_data_smithi032_140417_object100493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.074540+0000", 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.86728421700001, 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.015916855, 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:10:59.956 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233884 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074540+0000", 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074540+0000", 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.957 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074538+0000", 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074545+0000", 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2139999999999999e-06 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074558+0000", 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2448e-05 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.958 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074564+0000", 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6590000000000001e-06 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080211+0000", 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0056471289999999999 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080223+0000", 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1331e-05 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.959 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965420+0000", 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.88519726399999998 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965430+0000", 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0386e-05 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083497+0000", 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1180668620000001 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.960 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:10:59.961 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083583+0000", 2024-09-21T10:10:59.961 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6243999999999996e-05 2024-09-21T10:10:59.961 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.961 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.961 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:10:59.961 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083841+0000", 2024-09-21T10:10:59.961 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00025751600000000001 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086822+0000", 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002981048 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087068+0000", 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024563800000000001 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.962 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087100+0000", 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2191999999999999e-05 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090223+0000", 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00312277 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090262+0000", 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8892e-05 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.963 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090311+0000", 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9879999999999997e-05 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090457+0000", 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000145418 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233887 5.16 5:6ae9e2ce:::benchmark_data_smithi032_140417_object100496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:10:59.964 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.076252+0000", 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.865571915, 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0146629210000002, 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233887 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076252+0000", 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.965 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076252+0000", 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076251+0000", 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076257+0000", 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4929999999999998e-06 2024-09-21T10:10:59.966 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076271+0000", 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.451e-05 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076279+0000", 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2259999999999999e-06 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080245+0000", 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039659989999999996 2024-09-21T10:10:59.967 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080252+0000", 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8800000000000008e-06 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965457+0000", 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.88520482700000003 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965467+0000", 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.859e-06 2024-09-21T10:10:59.968 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.085168+0000", 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.11970049 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.085270+0000", 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000102038 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.085489+0000", 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000219848 2024-09-21T10:10:59.969 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087046+0000", 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001556102 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087128+0000", 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2212999999999994e-05 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087159+0000", 2024-09-21T10:10:59.970 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1071000000000001e-05 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090597+0000", 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0034384329999999999 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090768+0000", 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017031500000000001 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090801+0000", 2024-09-21T10:10:59.971 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3090999999999999e-05 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090915+0000", 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000114504 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233916 5.16 5:68361781:::benchmark_data_smithi032_140417_object100525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.104913+0000", 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.836911141, 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.986222589, 2024-09-21T10:10:59.972 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233916 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104913+0000", 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:10:59.973 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104913+0000", 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104911+0000", 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999976 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104918+0000", 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8340000000000001e-06 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.974 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104932+0000", 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4585e-05 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104939+0000", 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9709999999999998e-06 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.105003+0000", 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3701000000000006e-05 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.975 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.105024+0000", 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0996e-05 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965500+0000", 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.86047598400000003 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965510+0000", 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8509999999999994e-06 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087182+0000", 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.121672279 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087258+0000", 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5883000000000005e-05 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087464+0000", 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000206104 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090228+0000", 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002764168 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090519+0000", 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000290513 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090568+0000", 2024-09-21T10:10:59.978 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9011000000000002e-05 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091023+0000", 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00045542999999999999 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091049+0000", 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5609000000000002e-05 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091075+0000", 2024-09-21T10:10:59.979 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.6710999999999999e-05 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091136+0000", 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0202999999999999e-05 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234032 5.16 5:68dad721:::benchmark_data_smithi032_140417_object100641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.68289986800001, 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8323526370000001, 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:10:59.980 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234032 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.981 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258928+0000", 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8249999999999998e-06 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258933+0000", 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:10:59.982 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258937+0000", 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7699999999999999e-06 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258981+0000", 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3417000000000001e-05 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258988+0000", 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6680000000000001e-06 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.983 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965537+0000", 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.70654863199999995 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965548+0000", 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0576e-05 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.088645+0000", 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1230973209999999 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:10:59.984 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.088772+0000", 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012733500000000001 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.088991+0000", 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00021865900000000001 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090530+0000", 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015394339999999999 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090954+0000", 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00042421599999999999 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.985 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090993+0000", 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8735000000000001e-05 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091163+0000", 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017015199999999999 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.986 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.987 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091193+0000", 2024-09-21T10:10:59.987 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9189000000000001e-05 2024-09-21T10:10:59.987 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.987 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091216+0000", 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3572000000000001e-05 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091277+0000", 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0851999999999999e-05 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234034 5.16 5:6d7dbd1b:::benchmark_data_smithi032_140417_object100643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:10:59.988 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.677200934, 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.834288012, 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234034 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.989 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264627+0000", 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4209999999999997e-06 2024-09-21T10:10:59.990 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264638+0000", 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1372000000000001e-05 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264645+0000", 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6409999999999996e-06 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264695+0000", 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9638999999999997e-05 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264705+0000", 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0577999999999999e-05 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965574+0000", 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.70086844199999998 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965583+0000", 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8039999999999996e-06 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091314+0000", 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.125730994 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091445+0000", 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000131056 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091737+0000", 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00029182999999999998 2024-09-21T10:10:59.993 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098585+0000", 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0068481169999999999 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098634+0000", 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8315000000000001e-05 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098644+0000", 2024-09-21T10:10:59.994 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0589e-05 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098675+0000", 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0474999999999999e-05 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098715+0000", 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0546000000000001e-05 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098771+0000", 2024-09-21T10:10:59.995 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5798999999999999e-05 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098911+0000", 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000140128 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234086 5.16 5:68c88c0e:::benchmark_data_smithi032_140417_object100695:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.313647+0000", 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.62817704, 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.787847314, 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:10:59.996 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234086 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313647+0000", 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:10:59.997 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313647+0000", 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313646+0000", 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313652+0000", 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9190000000000004e-06 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:10:59.998 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313663+0000", 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1888999999999999e-05 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313669+0000", 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3619999999999997e-06 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313709+0000", 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0435999999999997e-05 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:10:59.999 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313719+0000", 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3100000000000006e-06 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965613+0000", 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.65189446500000003 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965623+0000", 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6530000000000006e-06 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.000 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.093795+0000", 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1281720960000001 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.093926+0000", 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013081799999999999 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.094285+0000", 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00035964900000000003 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.001 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098651+0000", 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0043658539999999997 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099003+0000", 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00035174099999999998 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099064+0000", 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1512000000000003e-05 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.002 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099119+0000", 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4803999999999999e-05 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101348+0000", 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022291329999999999 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101409+0000", 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0467000000000002e-05 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.003 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101494+0000", 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5681999999999997e-05 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234124 5.16 5:6c5cf963:::benchmark_data_smithi032_140417_object100733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.351979+0000", 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.589845411, 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.754470526, 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.004 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234124 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351979+0000", 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351979+0000", 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.005 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351978+0000", 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351982+0000", 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0729999999999998e-06 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351991+0000", 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4260000000000003e-06 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.006 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351996+0000", 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7219999999999996e-06 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.352067+0000", 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0650999999999998e-05 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.352078+0000", 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0991e-05 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.007 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965648+0000", 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.61357046599999998 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965657+0000", 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7949999999999996e-06 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.096078+0000", 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1304205760000001 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.008 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.096215+0000", 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000137248 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.096550+0000", 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00033544500000000002 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098924+0000", 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0023735240000000001 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.009 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101244+0000", 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00231985 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101293+0000", 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8739000000000002e-05 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101542+0000", 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024916499999999998 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.010 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106280+0000", 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0047382609999999997 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106326+0000", 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5986999999999999e-05 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106449+0000", 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000123297 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.011 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234166 5.16 5:6ca4e8a5:::benchmark_data_smithi032_140417_object100775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.387858+0000", 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.553965827, 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.718943055, 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234166 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.012 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.013 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:00.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387858+0000", 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387858+0000", 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.014 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387857+0000", 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387863+0000", 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4300000000000003e-06 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387877+0000", 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3878e-05 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.015 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387884+0000", 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8839999999999997e-06 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387933+0000", 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8795999999999998e-05 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387941+0000", 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3939999999999996e-06 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.016 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965685+0000", 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57774333 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965693+0000", 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9090000000000006e-06 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099158+0000", 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1334648899999999 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.017 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099306+0000", 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000148076 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099683+0000", 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00037678700000000001 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106241+0000", 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0065577659999999996 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.018 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106512+0000", 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000271301 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106522+0000", 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0155e-05 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106559+0000", 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6272000000000001e-05 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.019 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106608+0000", 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.914e-05 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106668+0000", 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9942000000000003e-05 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106801+0000", 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000133635 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.020 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234167 5.16 5:6ce4fd33:::benchmark_data_smithi032_140417_object100776:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.390340+0000", 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.55148436499999, 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.721460698, 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234167 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.021 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390340+0000", 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390340+0000", 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390339+0000", 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:00.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390347+0000", 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7729999999999999e-06 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390358+0000", 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1365999999999999e-05 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390363+0000", 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4949999999999999e-06 2024-09-21T10:11:00.023 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390416+0000", 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2979000000000002e-05 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390428+0000", 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1277000000000001e-05 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965719+0000", 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57529121699999997 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968363+0000", 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0026442810000000001 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101578+0000", 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.133214739 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101694+0000", 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115877 2024-09-21T10:11:00.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101921+0000", 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000227037 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106824+0000", 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049031949999999999 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106864+0000", 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9711e-05 2024-09-21T10:11:00.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106921+0000", 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7426999999999998e-05 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106960+0000", 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8778e-05 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111484+0000", 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0045242879999999996 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111533+0000", 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9286999999999998e-05 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111801+0000", 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00026706100000000002 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234170 5.16 5:6c81e8e9:::benchmark_data_smithi032_140417_object100779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.396582+0000", 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.54524217700001, 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.720816052, 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234170 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396582+0000", 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396582+0000", 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396581+0000", 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396585+0000", 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1250000000000003e-06 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396594+0000", 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8049999999999996e-06 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396598+0000", 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3730000000000003e-06 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.031 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396637+0000", 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8633000000000001e-05 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396654+0000", 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6682000000000001e-05 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968419+0000", 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57176563899999999 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.032 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968433+0000", 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3628000000000001e-05 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.103802+0000", 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.135369106 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.103915+0000", 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011266500000000001 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.033 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.104243+0000", 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00032865900000000003 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106878+0000", 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00263496 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107010+0000", 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013164200000000001 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111444+0000", 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0044338060000000002 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117054+0000", 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0056105570000000004 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117236+0000", 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000182058 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117282+0000", 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6072000000000002e-05 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117398+0000", 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115617 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234173 5.16 5:6d06a077:::benchmark_data_smithi032_140417_object100782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.399962+0000", 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.541862005, 2024-09-21T10:11:00.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.717757881, 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234173 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399962+0000", 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399962+0000", 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399961+0000", 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399966+0000", 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2000000000000002e-06 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399977+0000", 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0924000000000001e-05 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399982+0000", 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1399999999999999e-06 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.039 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.400024+0000", 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2184999999999997e-05 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.400032+0000", 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1709999999999996e-06 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.040 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968461+0000", 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.56842957699999996 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968471+0000", 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8339999999999994e-06 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107043+0000", 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1385719139999999 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107179+0000", 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013627299999999999 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107562+0000", 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00038266100000000001 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111861+0000", 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0042995639999999996 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111874+0000", 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2559e-05 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111932+0000", 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7599999999999997e-05 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117167+0000", 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0052358830000000002 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117589+0000", 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000421063 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117638+0000", 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9394999999999997e-05 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117720+0000", 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2128999999999999e-05 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.044 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234191 5.16 5:6e4c6783:::benchmark_data_smithi032_140417_object100800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.424857+0000", 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.516966916, 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.703213192, 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234191 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424857+0000", 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424857+0000", 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424856+0000", 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:00.046 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424862+0000", 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5010000000000001e-06 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424895+0000", 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2545000000000001e-05 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424908+0000", 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3281999999999999e-05 2024-09-21T10:11:00.047 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424983+0000", 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4461000000000004e-05 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424996+0000", 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3675999999999999e-05 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968498+0000", 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54350163500000004 2024-09-21T10:11:00.048 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968508+0000", 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0185e-05 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.109236+0000", 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.140728183 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.109352+0000", 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115758 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.109580+0000", 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00022780000000000001 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.114551+0000", 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049710209999999999 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117101+0000", 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0025498420000000001 2024-09-21T10:11:00.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117464+0000", 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036382999999999999 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117550+0000", 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5587999999999994e-05 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.127902+0000", 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010351463999999999 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.127950+0000", 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8401e-05 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128070+0000", 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120559 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234298 5.16 5:6cdfd838:::benchmark_data_smithi032_140417_object100907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.589148+0000", 2024-09-21T10:11:00.052 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.35267605600001, 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5391869969999998, 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234298 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589148+0000", 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.053 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589148+0000", 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589146+0000", 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589153+0000", 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8499999999999996e-06 2024-09-21T10:11:00.054 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589168+0000", 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.447e-05 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589174+0000", 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.003e-06 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589236+0000", 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2528999999999997e-05 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589249+0000", 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2418999999999999e-05 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968535+0000", 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.37928635399999999 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968545+0000", 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6390000000000004e-06 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111967+0000", 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.143422419 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.112085+0000", 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000117673 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.057 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.112431+0000", 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00034633500000000002 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117112+0000", 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0046805689999999999 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117503+0000", 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00039122000000000001 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117760+0000", 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00025673999999999998 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128141+0000", 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010381095999999999 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128180+0000", 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9677e-05 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128223+0000", 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2802000000000001e-05 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128335+0000", 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011193599999999999 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234305 5.16 5:6f4d2f3d:::benchmark_data_smithi032_140417_object100914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.745057+0000", 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.19676685500001, 2024-09-21T10:11:00.060 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3835431599999999, 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234305 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745057+0000", 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.061 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745057+0000", 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745056+0000", 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745061+0000", 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6789999999999996e-06 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745071+0000", 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.031e-05 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745076+0000", 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8060000000000002e-06 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745120+0000", 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3581999999999997e-05 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.063 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745129+0000", 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3030000000000005e-06 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968592+0000", 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.223463151 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968602+0000", 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0209e-05 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.114594+0000", 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1459913130000001 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.114739+0000", 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014554999999999999 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.115102+0000", 2024-09-21T10:11:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:10:59 smithi045 ceph-mon[95976]: pgmap v556: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-21T10:11:00.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036283700000000002 2024-09-21T10:11:00.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.066 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.066 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.066 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117514+0000", 2024-09-21T10:11:00.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024117330000000001 2024-09-21T10:11:00.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117800+0000", 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00028629399999999998 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117833+0000", 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2741999999999999e-05 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128400+0000", 2024-09-21T10:11:00.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010566862999999999 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128438+0000", 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8417000000000002e-05 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128488+0000", 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9842999999999997e-05 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128601+0000", 2024-09-21T10:11:00.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000112724 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234369 5.e 5:7f9b0c82:::benchmark_data_smithi032_140417_object100978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:05.101285+0000", 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 114.840539147, 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0605576990000001, 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.069 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234369 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101285+0000", 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101285+0000", 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.070 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101284+0000", 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101289+0000", 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.011e-06 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101298+0000", 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9730000000000003e-06 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101303+0000", 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1780000000000002e-06 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128798+0000", 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.027495070999999999 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128808+0000", 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5869999999999999e-06 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.149057+0000", 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0202487 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.151882+0000", 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002825634 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2,4", 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.152026+0000", 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014369799999999999 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.073 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161477+0000", 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094507810000000001 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161535+0000", 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8399000000000001e-05 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161592+0000", 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6943e-05 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161654+0000", 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2346000000000002e-05 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161711+0000", 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6950999999999997e-05 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161843+0000", 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013147699999999999 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.075 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234389 5.e 5:7b3804d8:::benchmark_data_smithi032_140417_object100998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:05.120077+0000", 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 114.821747538, 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0524572189999999, 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234389 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120077+0000", 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120077+0000", 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120076+0000", 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120080+0000", 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2039999999999999e-06 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120088+0000", 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6009999999999995e-06 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120094+0000", 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6540000000000004e-06 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128830+0000", 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087358999999999996 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128838+0000", 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3289999999999994e-06 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.155438+0000", 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0265995969999999 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.079 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.155529+0000", 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0626999999999998e-05 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2,4", 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.155640+0000", 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000111137 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161604+0000", 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0059642979999999998 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161886+0000", 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00028169099999999998 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161954+0000", 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8204e-05 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.172342+0000", 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010387828999999999 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.172409+0000", 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6789e-05 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.172534+0000", 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000125339 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.082 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234402 5.e 5:7aff253c:::benchmark_data_smithi032_140417_object101011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:05.131926+0000", 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 114.809898214, 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0452540100000001, 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234402 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131926+0000", 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131926+0000", 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131924+0000", 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131931+0000", 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0539999999999997e-06 2024-09-21T10:11:00.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131948+0000", 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.683e-05 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131957+0000", 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4989999999999994e-06 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.132233+0000", 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00027579200000000002 2024-09-21T10:11:00.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.132244+0000", 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1243999999999999e-05 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.158689+0000", 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0264447269999999 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.158781+0000", 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2371e-05 2024-09-21T10:11:00.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2,4", 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.158857+0000", 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5599000000000006e-05 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161892+0000", 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0030351319999999998 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.162017+0000", 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000125418 2024-09-21T10:11:00.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.168766+0000", 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0067484709999999998 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.176992+0000", 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0082261039999999997 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.177039+0000", 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7718e-05 2024-09-21T10:11:00.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.177180+0000", 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014060400000000001 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:238218 5.16 5:6e0183dc:::benchmark_data_smithi032_140417_object104827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:10.001585+0000", 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 109.940239564, 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.05259364, 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.089 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 238218 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001585+0000", 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001585+0000", 2024-09-21T10:11:00.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001584+0000", 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001588+0000", 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2710000000000001e-06 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001595+0000", 2024-09-21T10:11:00.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5279999999999998e-06 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001601+0000", 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7459999999999997e-06 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001629+0000", 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7766000000000001e-05 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001702+0000", 2024-09-21T10:11:00.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3644000000000002e-05 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.002088+0000", 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000385728 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.004328+0000", 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022396650000000001 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.004698+0000", 2024-09-21T10:11:00.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036994000000000003 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.004870+0000", 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017258400000000001 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.016872+0000", 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012001639 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.016887+0000", 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.508e-05 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:11.054100+0000", 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.037213476 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:11.054178+0000", 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7967999999999996e-05 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:242060 5.e 5:706bc916:::benchmark_data_smithi032_140417_object108669:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:16.562452+0000", 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 103.379372031, 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97550333600000005, 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 242060 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:00.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562452+0000", 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562452+0000", 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562450+0000", 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:00.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562457+0000", 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1189999999999999e-06 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562473+0000", 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5197e-05 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562479+0000", 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9739999999999999e-06 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562536+0000", 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7689999999999998e-05 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562547+0000", 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0577000000000001e-05 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.530619+0000", 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96807185799999995 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.530706+0000", 2024-09-21T10:11:00.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7257999999999997e-05 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.530781+0000", 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4865999999999998e-05 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.533900+0000", 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031187810000000002 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.533906+0000", 2024-09-21T10:11:00.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.708e-06 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.537894+0000", 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039874150000000002 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.537933+0000", 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9135999999999999e-05 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.537956+0000", 2024-09-21T10:11:00.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2603e-05 2024-09-21T10:11:00.102 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.102 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.102 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.102 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:00.102 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:00.102 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:11:00.114 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:00.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:00.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:00.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 38 16 71308 71292 117.243 71.0625 0.00645587 0.00852404 2024-09-21T10:11:00.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:11:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:10:59 smithi032 ceph-mon[105748]: pgmap v556: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-21T10:11:00.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:10:59 smithi032 ceph-mon[101299]: pgmap v556: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-21T10:11:00.248 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:00.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:00.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:00.383 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:00.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.417 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:00.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:00.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:00.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.556 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:00.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:00.657 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:00.692 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:00.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:00.794 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:00.831 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:00.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:00.932 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:00.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.955 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:00.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:01.056 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:01.093 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:01.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:01.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 39 16 72521 72505 116.18 75.8125 0.00886333 0.00860251 2024-09-21T10:11:01.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:01.236 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:01.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:01.238 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:01.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:11:01.338 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:01.403 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:01.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:01.505 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:01.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.623 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:01.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:01.724 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:01.771 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:01.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:01.893 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:01.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:01 smithi032 ceph-mon[101299]: pgmap v557: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-21T10:11:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:01 smithi032 ceph-mon[105748]: pgmap v557: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-21T10:11:02.004 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:02.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:02.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:02.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:02.145 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:01 smithi045 ceph-mon[95976]: pgmap v557: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-21T10:11:02.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:11:02.190268+0000 min lat: 0.00124686 max lat: 0.234546 avg lat: 0.00866165 2024-09-21T10:11:02.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:11:02.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 40 16 73871 73855 115.385 84.375 0.0073901 0.00866165 2024-09-21T10:11:02.194 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:02.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:02.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:02.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:02.366 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:02.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:02.468 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:02.507 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:02.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:02.609 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:02.649 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:02.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:02.751 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:02.789 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:02.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:02.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:02.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:03.030 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:03.070 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:03.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:03.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:03.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 41 16 75266 75250 114.697 87.1875 0.0177095 0.008714 2024-09-21T10:11:03.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.205 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:03.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:03.306 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:03.322 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:03.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.4 reset_purged_snaps_last 2024-09-21T10:11:03.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.355 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:03.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:03.457 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:03.496 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:03.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:03.597 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:03.650 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:03.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:03.786 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:03.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:03.876 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:03 smithi032 ceph-mon[105748]: pgmap v558: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:11:03.876 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:11:03.876 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:03.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:03 smithi032 ceph-mon[101299]: pgmap v558: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:11:03.876 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:11:03.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:03.889 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:03.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:03.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:04.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:03 smithi045 ceph-mon[95976]: pgmap v558: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:11:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:11:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:04.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.074 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:04.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:04.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:04.191 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 42 16 76371 76355 113.61 69.0625 0.0154045 0.00879667 2024-09-21T10:11:04.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.220 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:04.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:04.258 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76430 6.b e203/196 6:d0394d12:::benchmark_data_smithi032_171826_object76429:head v 203'4800, mlcod=203'4798)", 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.243384+0000", 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0077729339999999996, 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0078009250000000002, 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.243384+0000", 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.243384+0000", 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.243383+0000", 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.243414+0000", 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1109999999999999e-05 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.243423+0000", 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3829999999999998e-06 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.243429+0000", 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9920000000000004e-06 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.261 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.243485+0000", 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5884999999999998e-05 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.243503+0000", 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8050000000000002e-05 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.262 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:76436 6.1 6:8404d98d:::benchmark_data_smithi032_171826_object76435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e203)", 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.245019+0000", 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0061373529999999999, 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.006242063, 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 76436 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.245019+0000", 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.245019+0000", 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.245017+0000", 2024-09-21T10:11:04.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.245056+0000", 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8875000000000003e-05 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.245074+0000", 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7819999999999999e-05 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.245082+0000", 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6279999999999996e-06 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.245213+0000", 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000131738 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.245325+0000", 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000111564 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.245463+0000", 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013750500000000001 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248599+0000", 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031365220000000001 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.251092+0000", 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024931100000000002 2024-09-21T10:11:04.267 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76431 6.b e203/196 6:da262ca9:::benchmark_data_smithi032_171826_object76430:head v 203'4801, mlcod=203'4798)", 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.248026+0000", 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0031302869999999998, 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0033220020000000001, 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248026+0000", 2024-09-21T10:11:04.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248026+0000", 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248025+0000", 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248067+0000", 2024-09-21T10:11:04.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1541000000000001e-05 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248079+0000", 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1705999999999999e-05 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248085+0000", 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7009999999999999e-06 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248780+0000", 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00069446999999999998 2024-09-21T10:11:04.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248798+0000", 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8732999999999998e-05 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76433 6.c e203/193 6:3a859427:::benchmark_data_smithi032_171826_object76432:head v 203'4807, mlcod=203'4804)", 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.248098+0000", 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0030585579999999998, 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0033295680000000002, 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.271 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248098+0000", 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248098+0000", 2024-09-21T10:11:04.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248098+0000", 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248131+0000", 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3813000000000003e-05 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248138+0000", 2024-09-21T10:11:04.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4359999999999999e-06 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248142+0000", 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3250000000000001e-06 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248539+0000", 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00039636000000000001 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:04.274 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248558+0000", 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9227000000000001e-05 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76435 6.2 e203/193 6:456d2d97:::benchmark_data_smithi032_171826_object76434:head v 203'4768, mlcod=203'4766)", 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.248152+0000", 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0030044949999999998, 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003335649, 2024-09-21T10:11:04.275 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248152+0000", 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248152+0000", 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248152+0000", 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967296 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248186+0000", 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3769999999999997e-05 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248192+0000", 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9680000000000003e-06 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248196+0000", 2024-09-21T10:11:04.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0069999999999999e-06 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248243+0000", 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7465e-05 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248256+0000", 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2877999999999999e-05 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76423 6.3 e203/193 6:c9a3f5bc:::benchmark_data_smithi032_171826_object76422:head v 203'4824, mlcod=203'4822)", 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.248289+0000", 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0028678390000000001, 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0032529540000000002, 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248289+0000", 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.279 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248289+0000", 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248289+0000", 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967296 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248310+0000", 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1773e-05 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248316+0000", 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6069999999999998e-06 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248320+0000", 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0539999999999996e-06 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.251506+0000", 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031858120000000001 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.251523+0000", 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7688000000000001e-05 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76429 6.3 e203/193)", 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.248327+0000", 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0028295270000000001, 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0032657739999999999, 2024-09-21T10:11:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248327+0000", 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248327+0000", 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248327+0000", 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967296 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248346+0000", 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9415e-05 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248352+0000", 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6119999999999998e-06 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.248356+0000", 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8500000000000004e-06 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76439 6.c e203/193)", 2024-09-21T10:11:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.249345+0000", 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.001811661, 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022915679999999999, 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249345+0000", 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249345+0000", 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249344+0000", 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249386+0000", 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2116999999999997e-05 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249397+0000", 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0976e-05 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249403+0000", 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1689999999999997e-06 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76437 6.2 e203/193 6:420993cd:::benchmark_data_smithi032_171826_object76436:head v 203'4769, mlcod=203'4766)", 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.249410+0000", 2024-09-21T10:11:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0017470039999999999, 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002273869, 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249410+0000", 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249410+0000", 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249409+0000", 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249438+0000", 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9111e-05 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249445+0000", 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1680000000000001e-06 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.249449+0000", 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3359999999999997e-06 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.251053+0000", 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001604505 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.251074+0000", 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0775999999999999e-05 2024-09-21T10:11:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76443 6.d e203/196 6:b2438089:::benchmark_data_smithi032_171826_object76442:head v 203'4691, mlcod=203'4689)", 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.250400+0000", 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00075660199999999999, 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013383030000000001, 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250400+0000", 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250400+0000", 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250400+0000", 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250432+0000", 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2577000000000003e-05 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250439+0000", 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2520000000000002e-06 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250444+0000", 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.51e-06 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250742+0000", 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00029831299999999997 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250762+0000", 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0125000000000001e-05 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.44852.0:76440 6.c e203/193)", 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:04.250659+0000", 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00049734400000000004, 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001129609, 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250659+0000", 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250659+0000", 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250659+0000", 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250826+0000", 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016757299999999999 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250836+0000", 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4599999999999992e-06 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:04.250841+0000", 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8540000000000002e-06 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.299 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:04.300 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:11:04.300 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 11 2024-09-21T10:11:04.300 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:11:04.322 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:04.359 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:04.461 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:04.499 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:04.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:04.602 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:04.618 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:04.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:04.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:04.741 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:04.776 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:04.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:04.859 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:11:04.877 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:04.911 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:04.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:05.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:05.051 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:05.112 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:04 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3396239718' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:05.112 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:04 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:05.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:11:05.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:05.165 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:04 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3396239718' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:05.165 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:04 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:05.165 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:04 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3396239718' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:05.165 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:04 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:05.193 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 43 16 77652 77636 112.829 80.0625 0.0135503 0.00885767 2024-09-21T10:11:05.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:05.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:05.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.389 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:05.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:05.489 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:05.522 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:05.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:05.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:05.642 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:05.682 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:05.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:05.785 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:05.825 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:05.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:05.927 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:05.957 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:05 smithi045 ceph-mon[95976]: pgmap v559: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-21T10:11:05.958 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:05 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:05.958 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:05 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:05.958 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:05 smithi045 ceph-mon[95976]: osdmap e204: 8 total, 8 up, 7 in 2024-09-21T10:11:05.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:05.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:06.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:06.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:06.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:05 smithi032 ceph-mon[105748]: pgmap v559: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-21T10:11:06.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:05 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:06.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:05 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:06.110 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:05 smithi032 ceph-mon[105748]: osdmap e204: 8 total, 8 up, 7 in 2024-09-21T10:11:06.111 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:05 smithi032 ceph-mon[101299]: pgmap v559: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-21T10:11:06.111 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:05 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:06.111 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:05 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:06.111 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:05 smithi032 ceph-mon[101299]: osdmap e204: 8 total, 8 up, 7 in 2024-09-21T10:11:06.192 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 44 16 78952 78936 112.111 81.25 0.0184807 0.00891462 2024-09-21T10:11:06.197 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:06.237 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:06.340 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:06.386 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:06.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:06.488 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:06.521 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:06.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:06.623 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:06.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:06.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:06.758 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:06.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:06.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:06.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:06.935 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:06.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:07.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:07.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.085 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:07.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:07.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:07.192 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 45 16 80248 80232 111.419 81 0.007827 0.00896957 2024-09-21T10:11:07.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:07.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:07.323 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:07.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.380 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:07.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:07.481 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:07.520 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:07.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:07.581 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:11:07.581 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:11:07.581 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:11:07.582 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:11:07.582 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:11:07.622 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:07.669 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:07.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:07.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:07.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:07.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:07.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:07.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:07.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:07.959 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:07 smithi045 ceph-mon[95976]: pgmap v561: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:11:08.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:08.088 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:08.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:08.102 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:07 smithi032 ceph-mon[101299]: pgmap v561: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:11:08.102 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:07 smithi032 ceph-mon[105748]: pgmap v561: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:11:08.166 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:11:08.166 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:08.190 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:08.192 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 46 16 81496 81480 110.693 78 0.00850351 0.00902909 2024-09-21T10:11:08.196 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:11:08.196 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:08.225 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:11:08.225 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:08.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:08.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:08.260 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:11:08.260 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:08.292 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:11:08.292 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:08.329 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:11:08.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:08.332 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:11:08.333 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:08.369 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:11:08.370 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:08.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.371 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:08.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:08.400 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:11:08.400 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:08.472 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:08.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:08.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:08.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:08.653 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:08.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:08.693 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:08.755 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:08.801 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:08.903 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:08.942 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:08.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:09.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:11:09.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:09.059 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:08 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3553256990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:09.060 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:08 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3553256990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:09.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:09.121 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:08 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3553256990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:09.192 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 47 16 82702 82686 109.941 75.375 0.0051552 0.00909041 2024-09-21T10:11:09.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:09.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.242 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:09.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:09.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:09.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.388 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:09.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:09.489 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:09.535 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:09.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:09.636 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:09.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:09.780 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:09.815 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:09.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:09.846 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:11:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:11:09] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.51.0" 2024-09-21T10:11:09.917 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:09.957 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:10.058 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:10.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:10.161 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:09 smithi032 ceph-mon[105748]: pgmap v562: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:11:10.161 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:09 smithi032 ceph-mon[101299]: pgmap v562: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:11:10.176 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:09 smithi045 ceph-mon[95976]: pgmap v562: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:11:10.192 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 48 16 83987 83971 109.324 80.3125 0.0136348 0.00914156 2024-09-21T10:11:10.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.5 reset_purged_snaps_last 2024-09-21T10:11:10.230 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:10.299 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.317 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:10.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:10.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:10.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:10.562 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:10.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:10.701 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:10.701 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:10.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:10.740 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:10.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:10.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:10.879 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:10.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:10.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:11.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:11.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:11.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:11.152 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:11.193 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 49 16 85174 85158 108.606 74.1875 0.00897941 0.00920296 2024-09-21T10:11:11.254 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:11.296 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:11.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:11.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:11.435 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:11.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:11.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:11.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:11:11.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:11.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:11.676 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.718 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:11.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:11.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:11:11.811 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:11.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:11.883 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:11.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:11.885 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:11.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:12.017 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:11 smithi032 ceph-mon[101299]: pgmap v563: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-21T10:11:12.017 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:11 smithi032 ceph-mon[105748]: pgmap v563: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-21T10:11:12.029 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:12.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:12.131 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:12.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.174 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:12.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:12.193 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 50 16 86379 86363 107.94 75.3125 0.0135123 0.00925947 2024-09-21T10:11:12.275 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:12.280 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:11 smithi045 ceph-mon[95976]: pgmap v563: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-21T10:11:12.322 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:12.423 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:12.446 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:12.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:12.581 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:12.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:12.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:12.717 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:12.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:12.819 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:12.854 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:12.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:12.956 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:12.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.989 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:12.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:13.090 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:13.125 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:13.193 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 51 16 87511 87495 107.21 70.75 0.0156489 0.00932229 2024-09-21T10:11:13.228 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:13.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:13.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:13.373 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:13.414 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:13.516 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:13.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:13.657 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:13.694 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:13.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:13.795 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:13.838 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:13.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:13.871 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:11:13.940 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:13.981 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:13.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:14.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:14.092 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:13 smithi045 ceph-mon[95976]: pgmap v564: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:11:14.092 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3468616320' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:14.092 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:13 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:14.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:13 smithi032 ceph-mon[101299]: pgmap v564: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:11:14.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3468616320' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:14.124 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:13 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:14.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:13 smithi032 ceph-mon[105748]: pgmap v564: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:11:14.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3468616320' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:14.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:13 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.126 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:14.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:14.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 52 16 88718 88702 106.597 75.4375 0.0133614 0.00937623 2024-09-21T10:11:14.227 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:14.272 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:14.357 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:14.374 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:14.405 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:14.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:14.507 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:14.551 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:14.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:14.588 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:14.589 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:14.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.689 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:14.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:14.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:14.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.826 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:14.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:14.927 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:14.961 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:14.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:15.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:14 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:15.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:14 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:15.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:14 smithi045 ceph-mon[95976]: osdmap e205: 8 total, 8 up, 7 in 2024-09-21T10:11:15.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:15.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.099 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:15.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:15.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:14 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:15.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:14 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:15.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:14 smithi032 ceph-mon[105748]: osdmap e205: 8 total, 8 up, 7 in 2024-09-21T10:11:15.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:14 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:15.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:14 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:15.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:14 smithi032 ceph-mon[101299]: osdmap e205: 8 total, 8 up, 7 in 2024-09-21T10:11:15.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 53 16 89898 89882 105.977 73.75 0.011882 0.00943034 2024-09-21T10:11:15.200 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:15.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.241 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:15.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:15.323 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:11:15.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:15.375 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:15.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:15.477 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:15.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:15.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:15.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:15.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.651 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:15.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:15.702 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:15.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:15.784 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:15.886 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:15.919 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:16.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:16.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:15 smithi045 ceph-mon[95976]: pgmap v566: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s 2024-09-21T10:11:16.048 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:16.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:16.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:16.181 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:16.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:16.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 54 16 90915 90899 105.191 63.5625 0.0174786 0.00950093 2024-09-21T10:11:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:15 smithi032 ceph-mon[105748]: pgmap v566: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s 2024-09-21T10:11:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:15 smithi032 ceph-mon[101299]: pgmap v566: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s 2024-09-21T10:11:16.283 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:16.317 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:16.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:16.419 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:16.427 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.6 reset_purged_snaps_last 2024-09-21T10:11:16.451 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:16.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:16.553 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.597 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:16.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:16.698 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:16.734 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:16.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:16.836 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.878 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:16.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:16.932 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:11:16.979 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:17.032 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:17.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:17.134 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:17.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:17.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 55 16 91977 91961 104.485 66.375 0.0152817 0.00956587 2024-09-21T10:11:17.271 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:17.314 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:17.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:17.415 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:17.455 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:17.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:17.557 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:17.593 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:17.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:17.729 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:17.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:17.831 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:17.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.868 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:17.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:17.872 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:17.969 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:18.005 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:18.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:17 smithi045 ceph-mon[95976]: pgmap v567: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-21T10:11:18.091 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:17 smithi032 ceph-mon[101299]: pgmap v567: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-21T10:11:18.091 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:17 smithi032 ceph-mon[105748]: pgmap v567: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-21T10:11:18.107 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:18.143 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:18.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:18.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 56 16 93069 93053 103.838 68.25 0.015566 0.00962537 2024-09-21T10:11:18.244 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:18.269 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:18.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:18.371 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:18.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:18.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:18.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:18.547 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:18.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:18.648 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:18.683 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:18.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:18.715 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:18.785 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.822 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:18.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:18.923 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:18.967 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:18.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:19.068 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:19.083 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]} : dispatch 2024-09-21T10:11:19.083 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:11:19.083 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:19.083 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]} : dispatch 2024-09-21T10:11:19.083 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:11:19.083 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:19.120 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:19.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:19.148 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]} : dispatch 2024-09-21T10:11:19.148 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:11:19.148 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:19.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 57 16 94170 94154 103.224 68.8125 0.0051977 0.00968272 2024-09-21T10:11:19.222 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:19.261 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:19.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:19.363 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:19.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.400 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:19.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:19.470 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:11:19.470 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:11:19.470 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:11:19.471 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:19.471 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:19.471 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:19.471 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:11:19.471 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:11:19.471 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:11:19.472 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:19.472 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:19.472 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:19.486 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:11:19.486 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:19.500 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:19.521 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:11:19.521 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:19.551 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:19.603 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:11:19.603 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:11:19.603 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:11:19.604 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:11:19.604 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:11:19.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:19.690 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:19.691 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:11:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:11:19] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.51.0" 2024-09-21T10:11:19.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:19.835 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:19.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:19.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:19 smithi032 ceph-mon[105748]: pgmap v568: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:11:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]}]': finished 2024-09-21T10:11:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:19 smithi032 ceph-mon[105748]: osdmap e206: 8 total, 8 up, 7 in 2024-09-21T10:11:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4252868844' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:19 smithi032 ceph-mon[101299]: pgmap v568: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:11:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]}]': finished 2024-09-21T10:11:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:19 smithi032 ceph-mon[101299]: osdmap e206: 8 total, 8 up, 7 in 2024-09-21T10:11:19.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4252868844' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:19.972 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:20.074 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:20.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:20.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:20.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 58 16 95306 95290 102.668 71 0.00544916 0.00973461 2024-09-21T10:11:20.212 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.251 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:20.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:20.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:19 smithi045 ceph-mon[95976]: pgmap v568: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:11:20.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]}]': finished 2024-09-21T10:11:20.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:19 smithi045 ceph-mon[95976]: osdmap e206: 8 total, 8 up, 7 in 2024-09-21T10:11:20.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4252868844' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:20.321 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:11:20.352 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:20.382 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:20.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:20.484 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:20.524 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:20.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:20.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:20.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:20.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:20.739 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:20.758 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:20.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.805 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:20.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:20.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:20.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:20.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:21.050 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:21.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:21.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:21.112 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:20 smithi032 ceph-mon[105748]: osdmap e207: 8 total, 8 up, 7 in 2024-09-21T10:11:21.113 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:20 smithi032 ceph-mon[101299]: osdmap e207: 8 total, 8 up, 7 in 2024-09-21T10:11:21.171 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:20 smithi045 ceph-mon[95976]: osdmap e207: 8 total, 8 up, 7 in 2024-09-21T10:11:21.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:21.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 59 16 96400 96384 102.087 68.375 0.0135712 0.00979012 2024-09-21T10:11:21.205 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:21.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:21.307 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:21.345 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:21.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:21.446 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:21.483 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:21.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:21.585 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:21.614 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:21.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:21.716 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:21.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:21.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:21.856 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:21.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:21.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:21.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:22.041 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:22.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:22.143 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:22.181 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:22.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:22.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:11:22.194434+0000 min lat: 0.00124686 max lat: 0.234546 avg lat: 0.00983641 2024-09-21T10:11:22.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:11:22.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 60 16 97575 97559 101.609 73.4375 0.00655552 0.00983641 2024-09-21T10:11:22.201 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:21 smithi045 ceph-mon[95976]: pgmap v571: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-21T10:11:22.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:21 smithi032 ceph-mon[105748]: pgmap v571: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-21T10:11:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:21 smithi032 ceph-mon[101299]: pgmap v571: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-21T10:11:22.283 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:22.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:22.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:22.425 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:22.482 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:22.583 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:22.613 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:22.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:22.660 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:11:22.660 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:11:22.660 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:11:22.660 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:11:22.660 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:26440 5.9 5:94853574:::benchmark_data_smithi032_140417_object26439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:05:38.493569+0000", 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 344.16139437300001, 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.164516671, 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 26440 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.661 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493569+0000", 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493569+0000", 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493567+0000", 2024-09-21T10:11:22.662 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493668+0000", 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010119999999999999 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493683+0000", 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4228e-05 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.663 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493691+0000", 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8140000000000001e-06 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493802+0000", 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000111934 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493948+0000", 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000145432 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.664 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.494104+0000", 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000156205 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.495291+0000", 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001186566 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.665 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.495682+0000", 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00039149900000000001 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.657974+0000", 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16229226399999999 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.658034+0000", 2024-09-21T10:11:22.666 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9431000000000001e-05 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.658086+0000", 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1699000000000001e-05 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:26451 5.9 5:9a35ac15:::benchmark_data_smithi032_140417_object26450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:05:38.496388+0000", 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 344.15857520999998, 2024-09-21T10:11:22.667 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.165390493, 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 26451 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496388+0000", 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.668 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496388+0000", 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496387+0000", 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496537+0000", 2024-09-21T10:11:22.669 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014985899999999999 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496548+0000", 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1331e-05 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496554+0000", 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9880000000000001e-06 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496596+0000", 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2224000000000001e-05 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496729+0000", 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013331700000000001 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496861+0000", 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013177999999999999 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.671 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.497956+0000", 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001095279 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498289+0000", 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00033209500000000002 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661693+0000", 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16340427299999999 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.672 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661743+0000", 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0563999999999999e-05 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661779+0000", 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5138999999999997e-05 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:26453 5.9 5:9800755e:::benchmark_data_smithi032_140417_object26452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:05:38.498323+0000", 2024-09-21T10:11:22.673 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 344.156640133, 2024-09-21T10:11:22.674 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16356768499999999, 2024-09-21T10:11:22.674 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.674 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:22.674 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.674 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.674 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.674 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 26453 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498323+0000", 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498323+0000", 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.675 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498322+0000", 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498394+0000", 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1762999999999997e-05 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498408+0000", 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3726999999999999e-05 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498413+0000", 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.784e-06 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498441+0000", 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7759e-05 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498528+0000", 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6885000000000006e-05 2024-09-21T10:11:22.677 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498608+0000", 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9900000000000004e-05 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.499458+0000", 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00084979700000000001 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:22.678 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.499723+0000", 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000265159 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661818+0000", 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16209504399999999 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661859+0000", 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0960000000000001e-05 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661891+0000", 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1968000000000003e-05 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213387 5.10 5.645680d0 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.883648+0000", 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.77131493100001, 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99656069300000005, 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.680 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213387 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883648+0000", 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883648+0000", 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.681 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883646+0000", 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883654+0000", 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2479999999999999e-06 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883669+0000", 2024-09-21T10:11:22.682 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5434000000000002e-05 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883677+0000", 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6799999999999993e-06 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883759+0000", 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2169999999999994e-05 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883767+0000", 2024-09-21T10:11:22.683 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1629999999999996e-06 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880198+0000", 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99643097999999997 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880209+0000", 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0875e-05 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.684 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213398 5.10 5.bd5fbbf0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.892190+0000", 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.76277336300001, 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98813121900000001, 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213398 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.685 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892190+0000", 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892190+0000", 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892189+0000", 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.686 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892194+0000", 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9169999999999997e-06 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892205+0000", 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1073000000000001e-05 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892213+0000", 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9379999999999992e-06 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892273+0000", 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0674999999999998e-05 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892284+0000", 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0335e-05 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880308+0000", 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98802432900000003 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.688 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880321+0000", 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3022e-05 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213409 5.10 5.e3d5e190 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.899439+0000", 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.75552393800001, 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98091395400000003, 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.689 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213409 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899439+0000", 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899439+0000", 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899438+0000", 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899443+0000", 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8749999999999999e-06 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899454+0000", 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1017e-05 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899461+0000", 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.391e-06 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899513+0000", 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2345000000000001e-05 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899522+0000", 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0590000000000003e-06 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.692 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880345+0000", 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98082366899999995 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880353+0000", 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0569999999999993e-06 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213416 5.10 5.10f69050 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.905431+0000", 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.749531937, 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97496326499999997, 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213416 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905431+0000", 2024-09-21T10:11:22.694 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905431+0000", 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905430+0000", 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905438+0000", 2024-09-21T10:11:22.695 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0849999999999997e-06 2024-09-21T10:11:22.696 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.696 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.696 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.696 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905450+0000", 2024-09-21T10:11:22.696 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1790000000000001e-05 2024-09-21T10:11:22.697 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.697 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.697 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905456+0000", 2024-09-21T10:11:22.697 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.19e-06 2024-09-21T10:11:22.697 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905491+0000", 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4449999999999997e-05 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905495+0000", 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6099999999999999e-06 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880383+0000", 2024-09-21T10:11:22.698 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97488764400000005 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880395+0000", 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.154e-05 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213453 5.10 5.4c1650d0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.939270+0000", 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.71569289600001, 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.94115652100000002, 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.699 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213453 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939270+0000", 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939270+0000", 2024-09-21T10:11:22.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939269+0000", 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939275+0000", 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7909999999999999e-06 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939291+0000", 2024-09-21T10:11:22.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5420000000000001e-05 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939296+0000", 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3290000000000001e-06 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939341+0000", 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4842000000000002e-05 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939350+0000", 2024-09-21T10:11:22.703 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8880000000000003e-06 2024-09-21T10:11:22.703 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.703 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.703 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.703 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880416+0000", 2024-09-21T10:11:22.703 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.94106558200000001 2024-09-21T10:11:22.703 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880427+0000", 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1365e-05 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213458 5.10 5.aa724650 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.947131+0000", 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.70783204, 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.93332198799999999, 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.704 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213458 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947131+0000", 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947131+0000", 2024-09-21T10:11:22.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947130+0000", 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947136+0000", 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8250000000000001e-06 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947145+0000", 2024-09-21T10:11:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2369999999999998e-06 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947152+0000", 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.64e-06 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947195+0000", 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2960000000000002e-05 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947203+0000", 2024-09-21T10:11:22.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2269999999999993e-06 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880444+0000", 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.93324136300000005 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880453+0000", 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8200000000000003e-06 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213507 5.10 5.fbac80f0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.985956+0000", 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.66900755099999, 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.89453823399999999, 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213507 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.709 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985956+0000", 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985956+0000", 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985955+0000", 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.710 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985960+0000", 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3979999999999998e-06 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985975+0000", 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4622e-05 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985983+0000", 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9179999999999994e-06 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.711 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.986035+0000", 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2018999999999998e-05 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.986042+0000", 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4869999999999996e-06 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880485+0000", 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.89444252199999996 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.712 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880494+0000", 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3680000000000008e-06 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213582 5.10 5.ab58c590 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.048306+0000", 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.60665709200001, 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.83221718, 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.713 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213582 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048306+0000", 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048306+0000", 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.714 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048305+0000", 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048310+0000", 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9150000000000004e-06 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048321+0000", 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0861e-05 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048327+0000", 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6370000000000002e-06 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048387+0000", 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9551e-05 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048394+0000", 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.277e-06 2024-09-21T10:11:22.716 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880518+0000", 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.83212388800000003 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880523+0000", 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2519999999999999e-06 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213626 5.10 5.4cd189b0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.084187+0000", 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.570776139, 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.79637013700000003, 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213626 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.718 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084187+0000", 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084187+0000", 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084186+0000", 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.719 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084191+0000", 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4729999999999999e-06 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084202+0000", 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0596e-05 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084209+0000", 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0360000000000001e-06 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.720 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084256+0000", 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7663000000000001e-05 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084264+0000", 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0220000000000004e-06 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880549+0000", 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.79628444099999995 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.721 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880557+0000", 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3350000000000007e-06 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213633 5.10 5.18cc6f50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.092236+0000", 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.562727373, 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.78834528699999995, 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.722 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213633 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092236+0000", 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092236+0000", 2024-09-21T10:11:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092234+0000", 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092243+0000", 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4630000000000007e-06 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092259+0000", 2024-09-21T10:11:22.724 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5245000000000001e-05 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092267+0000", 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6300000000000004e-06 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092311+0000", 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3951e-05 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092319+0000", 2024-09-21T10:11:22.725 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7640000000000005e-06 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880576+0000", 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.78825735200000002 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880581+0000", 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7190000000000001e-06 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.726 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213651 5.10 5.b09713b0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.108041+0000", 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.54692192900001, 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.77257185100000003, 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213651 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.727 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108041+0000", 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108041+0000", 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108040+0000", 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.728 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108045+0000", 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4750000000000004e-06 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108074+0000", 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9459999999999999e-05 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108086+0000", 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2035000000000001e-05 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.729 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.730 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.730 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108144+0000", 2024-09-21T10:11:22.730 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8010000000000002e-05 2024-09-21T10:11:22.730 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.730 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.730 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.730 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:22.731 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108153+0000", 2024-09-21T10:11:22.731 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0830000000000004e-06 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880605+0000", 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.772452162 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880613+0000", 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4419999999999998e-06 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.732 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213671 5.10 5.775f8bb0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.127502+0000", 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.52746169900001, 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.75312580900000003, 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213671 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.733 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127502+0000", 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127502+0000", 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127501+0000", 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:22.734 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127505+0000", 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6770000000000003e-06 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127516+0000", 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0674e-05 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127521+0000", 2024-09-21T10:11:22.735 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8060000000000002e-06 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127557+0000", 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6684999999999999e-05 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127562+0000", 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9509999999999997e-06 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880623+0000", 2024-09-21T10:11:22.736 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.75306079199999998 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880627+0000", 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.245e-06 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213799 5.10 5.20d823f0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.279962+0000", 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.37500083800001, 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.60068750100000001, 2024-09-21T10:11:22.737 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213799 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279962+0000", 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.738 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279962+0000", 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279961+0000", 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279967+0000", 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.665e-06 2024-09-21T10:11:22.739 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279978+0000", 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1405e-05 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279985+0000", 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.652e-06 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.280054+0000", 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8783999999999997e-05 2024-09-21T10:11:22.740 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.280062+0000", 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9780000000000005e-06 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880642+0000", 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.60057990299999997 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880650+0000", 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2979999999999992e-06 2024-09-21T10:11:22.741 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213835 5.10 5.b1c1bcf0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.339730+0000", 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.31523295400001, 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54093409000000003, 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.742 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213835 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339730+0000", 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339730+0000", 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.743 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339729+0000", 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339736+0000", 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9240000000000001e-06 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339757+0000", 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1168000000000001e-05 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.744 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339766+0000", 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7649999999999999e-06 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339824+0000", 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7893000000000003e-05 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339833+0000", 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.628e-06 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.745 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880660+0000", 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54082699199999995 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880664+0000", 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.233e-06 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213850 5.10 5.b9b1c390 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.365015+0000", 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.28994780900001, 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.515670454, 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213850 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365015+0000", 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.747 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365015+0000", 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365014+0000", 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365021+0000", 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6640000000000001e-06 2024-09-21T10:11:22.748 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365033+0000", 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2131e-05 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365038+0000", 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5319999999999997e-06 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365082+0000", 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3278999999999997e-05 2024-09-21T10:11:22.749 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365090+0000", 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2150000000000001e-06 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880677+0000", 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.51558764300000004 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880686+0000", 2024-09-21T10:11:22.750 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4400000000000005e-06 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213854 5.10 5.6b4dc610 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.376078+0000", 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 169.278885538, 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.50462188200000002, 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:11:22.751 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213854 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376078+0000", 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376078+0000", 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.752 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376077+0000", 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376081+0000", 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2540000000000003e-06 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376090+0000", 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0000000000000002e-06 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.753 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376094+0000", 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.98e-06 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376130+0000", 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6358e-05 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376136+0000", 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6339999999999998e-06 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880695+0000", 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.50455955799999996 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880700+0000", 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1620000000000001e-06 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:39523 6.3 6:c70b7cc0:::benchmark_data_smithi032_171826_object39522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-21T10:11:22.755 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:10:38.364862+0000", 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 44.290101528999998, 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16180813299999999, 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 39523 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.364862+0000", 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.756 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.364862+0000", 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.364861+0000", 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.364889+0000", 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8073e-05 2024-09-21T10:11:22.757 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.364897+0000", 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6329999999999996e-06 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.364903+0000", 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7939999999999999e-06 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.364971+0000", 2024-09-21T10:11:22.758 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8238999999999995e-05 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.365075+0000", 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010432799999999999 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.366793+0000", 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001718074 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.514708+0000", 2024-09-21T10:11:22.759 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14791435999999999 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.514902+0000", 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00019467599999999999 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.526588+0000", 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011686017999999999 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.526635+0000", 2024-09-21T10:11:22.760 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6249999999999999e-05 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:10:38.526670+0000", 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5163999999999997e-05 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:11:22.761 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:11:22.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.763 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:22.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:22.864 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:22.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:23.005 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:23.048 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:23.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:23.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:23.189 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:23.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 61 16 98630 98614 101.024 65.9375 0.00688852 0.00989361 2024-09-21T10:11:23.291 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:23.331 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:23.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:23.433 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:23.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:23.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:23.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:23.604 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:23.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:23.706 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:23.729 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:23.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:23.831 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:23.863 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:23.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:23.965 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.001 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:24.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:24.102 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.132 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:24.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:24.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 62 16 99574 99558 100.346 59 0.00816607 0.00996036 2024-09-21T10:11:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:23 smithi032 ceph-mon[105748]: pgmap v572: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:11:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:23 smithi032 ceph-mon[101299]: pgmap v572: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:11:24.234 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:24.268 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:24.271 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:11:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:23 smithi045 ceph-mon[95976]: pgmap v572: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:11:24.370 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:24.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.415 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:24.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:24.516 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:24.551 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:24.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:24.653 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:24.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:24.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:24.834 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:24.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:24.937 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:24.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.966 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:24.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:25.067 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:25.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:25.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:25.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 63 16 100656 100640 99.8269 67.625 0.00490133 0.0100105 2024-09-21T10:11:25.209 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:25.247 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:25.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:25.260 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.7 reset_purged_snaps_last 2024-09-21T10:11:25.327 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:11:25.349 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:25.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.387 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:25.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:25.488 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:25.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.579 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:25.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:25.605 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:25.680 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:25.733 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:25.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:25.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:25.878 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:25.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:25.980 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:26.026 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:26.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:26.128 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:26.151 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:25 smithi032 ceph-mon[101299]: pgmap v573: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-21T10:11:26.151 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:25 smithi032 ceph-mon[105748]: pgmap v573: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-21T10:11:26.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:26.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:26.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 64 16 101659 101643 99.2465 62.6875 0.0160202 0.0100712 2024-09-21T10:11:26.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:25 smithi045 ceph-mon[95976]: pgmap v573: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-21T10:11:26.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.309 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:26.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:26.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:26.444 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:26.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:26.546 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:26.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:26.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:26.676 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:26.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:26.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:26.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:26.856 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:26.949 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:11:26.957 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:27.002 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:27.105 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:27.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:27.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 65 16 102778 102762 98.7955 69.9375 0.0042722 0.0101126 2024-09-21T10:11:27.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3225850204' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:11:27.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3225850204' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:11:27.248 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:27.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:27.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3225850204' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:11:27.384 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:27.420 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:27.522 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:27.560 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:27.661 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:27.695 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:27.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:27.797 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:27.829 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:27.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:27.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.968 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:27.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:28.068 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:28.104 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:28.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:28.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 66 16 103828 103812 98.2927 65.625 0.0198692 0.0101603 2024-09-21T10:11:28.207 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:28.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:27 smithi032 ceph-mon[105748]: pgmap v574: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-21T10:11:28.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:27 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:28.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3225850204' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:11:28.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:27 smithi032 ceph-mon[105748]: osdmap e208: 8 total, 8 up, 7 in 2024-09-21T10:11:28.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:27 smithi032 ceph-mon[101299]: pgmap v574: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-21T10:11:28.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:27 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:28.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3225850204' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:11:28.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:27 smithi032 ceph-mon[101299]: osdmap e208: 8 total, 8 up, 7 in 2024-09-21T10:11:28.248 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:28.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:27 smithi045 ceph-mon[95976]: pgmap v574: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-21T10:11:28.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:27 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:28.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3225850204' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:11:28.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:27 smithi045 ceph-mon[95976]: osdmap e208: 8 total, 8 up, 7 in 2024-09-21T10:11:28.350 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.395 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:28.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:28.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:28.532 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:28.634 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:28.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:28.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:28.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:28.803 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:28.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:28.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:28.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.942 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:28.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:29.043 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:29.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.078 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:29.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:29.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:29.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 67 16 104919 104903 97.8433 68.1875 0.00973183 0.0102154 2024-09-21T10:11:29.215 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:29.317 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:29.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.355 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:29.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:29.456 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.495 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:29.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:29.596 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:29.634 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:29.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:29.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:29.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:29.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:29.879 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:29.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:29.944 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:11:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:11:29] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.51.0" 2024-09-21T10:11:30.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:30.063 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:30.164 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:30.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:30.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:30.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 68 16 105939 105923 97.3419 63.75 0.00722911 0.0102681 2024-09-21T10:11:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:29 smithi032 ceph-mon[105748]: pgmap v576: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 857 op/s 2024-09-21T10:11:30.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:29 smithi032 ceph-mon[101299]: pgmap v576: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 857 op/s 2024-09-21T10:11:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:29 smithi045 ceph-mon[95976]: pgmap v576: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 857 op/s 2024-09-21T10:11:30.296 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:30.335 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:30.438 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:30.480 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:30.581 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:30.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:30.726 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:30.726 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:30.796 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:30.898 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:30.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.931 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:30.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:31.032 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:31.075 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:31.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:31.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 69 16 107017 107001 96.9075 67.375 0.0109681 0.0103143 2024-09-21T10:11:31.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:31.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:31.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:31.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:31.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:31.439 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:31.473 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:31.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:31.575 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:31.612 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:31.714 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:31.753 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:31.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:31.855 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:31.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:31.997 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:32.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.041 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:32.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:32.142 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:32.180 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:32.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:32.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 70 16 108144 108128 96.5293 70.4375 0.00804989 0.0103547 2024-09-21T10:11:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:31 smithi032 ceph-mon[101299]: pgmap v577: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:32.240 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:31 smithi032 ceph-mon[105748]: pgmap v577: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:32.282 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:31 smithi045 ceph-mon[95976]: pgmap v577: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:32.317 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:32.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:32.419 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:32.460 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:32.561 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:32.600 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:32.701 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:32.741 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:32.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:32.884 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:32.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:33.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.024 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:33.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:33.125 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:33.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:33.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:33.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 71 16 109278 109262 96.1679 70.875 0.0122035 0.0103929 2024-09-21T10:11:33.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:33.298 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:33.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:33.400 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:33.438 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:33.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:33.540 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:33.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.577 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:33.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:33.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:33.714 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:33.816 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:33.858 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:33.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:33.959 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:33.994 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:33.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:34.096 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:34.114 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:33 smithi032 ceph-mon[105748]: pgmap v578: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:11:34.114 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:34.114 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:33 smithi032 ceph-mon[101299]: pgmap v578: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:11:34.114 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:34.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.132 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:34.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:34.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 72 16 110428 110412 95.8303 71.875 0.00646782 0.0104302 2024-09-21T10:11:34.234 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:34.270 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:34.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:33 smithi045 ceph-mon[95976]: pgmap v578: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:11:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:34.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:34.411 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:34.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:34.471 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:11:34.471 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:34.506 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:11:34.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:34.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:34.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:11:34.548 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:11:34.548 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:34.558 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:34.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:34.584 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:11:34.584 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:34.619 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:11:34.619 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:34.653 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:11:34.654 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:34.660 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:34.692 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:11:34.692 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:34.694 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:34.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:34.733 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:11:34.733 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:34.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:34.835 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:34.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:34.937 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:34.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:34.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:35.077 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:35.131 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:35.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:35.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 73 16 111517 111501 95.449 68.0625 0.0111623 0.0104723 2024-09-21T10:11:35.233 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.276 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:35.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1409542736' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:35.378 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:35.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:35.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:35.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1409542736' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1409542736' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:35.511 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:35.547 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:35.648 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:35.686 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:35.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:35.788 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:35.828 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:35.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:35.929 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:35.964 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:36.066 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:36.101 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:36.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:36.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 74 16 112638 112622 95.1059 70.0625 0.00607831 0.0105093 2024-09-21T10:11:36.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:36.225 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:35 smithi032 ceph-mon[105748]: pgmap v579: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:11:36.225 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:35 smithi032 ceph-mon[101299]: pgmap v579: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:11:36.235 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:36.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:35 smithi045 ceph-mon[95976]: pgmap v579: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:11:36.337 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:36.370 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:36.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:36.472 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.508 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:36.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:36.609 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.653 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:36.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:36.754 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:36.789 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:36.891 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:36.929 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:36.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:36.950 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:36.951 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:37.031 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:37.069 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:37.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:37.171 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:37.201 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 75 16 113706 113690 94.7278 66.75 0.0253719 0.0105518 2024-09-21T10:11:37.206 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:37.308 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:37.343 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:37.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:37.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:37.482 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:37.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:37.584 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:37.619 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:37.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:37.721 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:37.758 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:37.807 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:11:37.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:37.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.925 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:37.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:37.933 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:38.027 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.067 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:38.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:38.072 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:37 smithi032 ceph-mon[101299]: pgmap v580: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-21T10:11:38.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:37 smithi032 ceph-mon[105748]: pgmap v580: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-21T10:11:38.168 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:38.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 76 16 114800 114784 94.381 68.375 0.0103409 0.0105904 2024-09-21T10:11:38.207 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:38.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:37 smithi045 ceph-mon[95976]: pgmap v580: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-21T10:11:38.309 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:38.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:38.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:38.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.477 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:38.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:38.577 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:38.611 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:38.688 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:11:38.713 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:38.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.771 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:38.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:38.851 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:11:38.851 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:11:38.852 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-09-21T10:11:38.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.0 reset_purged_snaps_last 2024-09-21T10:11:38.873 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:38.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:39.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.093 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:39.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:39.194 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:39.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 77 16 115896 115880 94.0448 68.5 0.0111339 0.0106285 2024-09-21T10:11:39.238 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:39.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:39.340 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:39.364 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:39.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:39.466 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:39.508 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:39.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:39.610 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:39.638 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:39.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:39.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:39.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:39.788 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:11:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:11:39] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.51.0" 2024-09-21T10:11:39.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:39.889 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:39.932 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:39.966 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:39.966 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:40.034 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:40.090 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:40.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:40.106 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:39 smithi032 ceph-mon[101299]: pgmap v581: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-21T10:11:40.106 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:39 smithi032 ceph-mon[105748]: pgmap v581: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-21T10:11:40.192 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:40.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 78 16 116937 116921 93.6732 65.0625 0.0144656 0.0106706 2024-09-21T10:11:40.225 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:40.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:39 smithi045 ceph-mon[95976]: pgmap v581: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-21T10:11:40.327 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:40.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.370 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:40.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:40.471 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:40.509 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:40.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:40.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:40.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:40.757 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:40.793 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:40.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:40.894 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:40.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:40.990 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:11:41.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:41.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:41.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:41.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:41.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 79 16 118047 118031 93.3656 69.375 0.0220887 0.0107052 2024-09-21T10:11:41.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:40 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/675830504' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:41.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:40 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:40 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/675830504' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:40 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:41.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:41.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:40 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/675830504' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:40 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:41.310 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:41.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.355 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:41.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:41.456 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:41.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:41.589 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:41.626 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:41.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:41.728 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:41.760 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:41.862 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:41.903 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:41.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:42.006 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:42.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.043 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:42.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:42.144 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:42.185 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:42.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:42.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:11:42.197037+0000 min lat: 0.00124686 max lat: 0.234546 avg lat: 0.0107371 2024-09-21T10:11:42.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:11:42.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 80 16 119192 119176 93.093 71.5625 0.0138851 0.0107371 2024-09-21T10:11:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:41 smithi045 ceph-mon[95976]: pgmap v582: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:11:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:41 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:11:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:41 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:11:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:41 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:41 smithi045 ceph-mon[95976]: osdmap e209: 8 total, 8 up, 7 in 2024-09-21T10:11:42.287 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:42.287 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[101299]: pgmap v582: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:11:42.287 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:11:42.288 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:11:42.288 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:42.288 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[101299]: osdmap e209: 8 total, 8 up, 7 in 2024-09-21T10:11:42.288 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[105748]: pgmap v582: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:11:42.288 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:11:42.288 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:11:42.288 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:42.288 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:41 smithi032 ceph-mon[105748]: osdmap e209: 8 total, 8 up, 7 in 2024-09-21T10:11:42.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.329 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:42.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:42.430 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:42.468 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:42.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:42.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:42.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:42.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:42.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.752 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:42.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:42.853 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:42.890 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:42.992 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:43.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:43.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:43.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.168 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:43.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:43.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 81 16 120306 120290 92.8032 69.625 0.0150094 0.0107709 2024-09-21T10:11:43.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:43.302 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:43.404 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:43.438 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:43.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:43.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:43.579 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:43.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:43.681 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:43.720 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:43.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:43.822 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:43.849 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:43.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:43.951 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:43.991 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:44.092 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:44.137 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:44.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:44.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 82 16 121394 121378 92.5006 68 0.0155539 0.0108057 2024-09-21T10:11:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:43 smithi032 ceph-mon[105748]: pgmap v584: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:11:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:43 smithi032 ceph-mon[101299]: pgmap v584: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:11:44.238 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:44.275 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:44.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:44.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:43 smithi045 ceph-mon[95976]: pgmap v584: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:11:44.377 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:44.416 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:44.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:44.559 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:44.661 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:44.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.705 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:44.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:44.806 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:44.839 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:44.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:44.940 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:44.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:44.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:45.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:45.119 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:45.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 83 15 122348 122333 92.1052 59.6875 0.00471709 0.010852 2024-09-21T10:11:45.221 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:45.246 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:45.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:45.348 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:45.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:45.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:45.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:45.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.522 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:45.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:45.623 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:45.651 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:45.753 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:45.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.794 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:45.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:45.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.934 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:45.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:46.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:46.066 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:46.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:46.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:46.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 84 16 123298 123282 91.7148 59.3125 0.00968781 0.0108988 2024-09-21T10:11:46.203 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:46.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:45 smithi032 ceph-mon[101299]: pgmap v585: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:45 smithi032 ceph-mon[105748]: pgmap v585: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:45 smithi045 ceph-mon[95976]: pgmap v585: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:46.305 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:46.341 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:46.443 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:46.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:46.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:46.501 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:11:46.501 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:46.534 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:11:46.535 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:46.567 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:11:46.567 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:46.582 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:46.600 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:11:46.600 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.613 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:46.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:46.650 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:11:46.651 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:46.692 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:11:46.692 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:46.714 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:46.739 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:11:46.739 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:46.741 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:46.758 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:11:46.758 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:11:46.758 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:11:46.758 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:11:46.758 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:11:46.772 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:11:46.773 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:46.785 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:11:46.843 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:46.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:46.995 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:47.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:47.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:47.138 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:47.172 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:47.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:47.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 85 16 124347 124331 91.4071 65.5625 0.0108078 0.0109352 2024-09-21T10:11:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:47 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1468493921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:47 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1468493921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:47.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:47.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:47 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1468493921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:47.309 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:47.341 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.2 reset_purged_snaps_last 2024-09-21T10:11:47.411 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:47.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.452 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:47.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:47.553 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:47.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.610 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:47.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:47.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:47.749 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:47.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:47.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:47.989 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:48.027 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:48.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:48.129 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:48.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:48.198 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 86 16 125427 125411 91.1291 67.5 0.0151349 0.0109685 2024-09-21T10:11:48.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:48 smithi045 ceph-mon[95976]: pgmap v586: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:11:48.304 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:48.406 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:48.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.444 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:48.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:48.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:48 smithi032 ceph-mon[101299]: pgmap v586: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:11:48.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:48 smithi032 ceph-mon[105748]: pgmap v586: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:11:48.545 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:48.580 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:48.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:48.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:48.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:48.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:48.822 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:48.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.859 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:48.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:48.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:11:48.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:48.962 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:49.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:49.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:49.039 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:49.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:49.171 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:49.199 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 87 16 126503 126487 90.8543 67.25 0.0112773 0.0110016 2024-09-21T10:11:49.273 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:49.331 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:49.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:49.433 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:49.470 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:49.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:49.572 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:49.607 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:49.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:11:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:11:49] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.51.0" 2024-09-21T10:11:49.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:49 smithi032 ceph-mon[105748]: pgmap v587: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:49.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:11:49.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:49.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:49 smithi032 ceph-mon[101299]: pgmap v587: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:49.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:11:49.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:49.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:49.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:49.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:49 smithi045 ceph-mon[95976]: pgmap v587: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:11:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:49.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:49.888 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:49.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:49.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:50.028 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:50.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:50.129 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:50.165 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:50.183 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:11:50.201 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 88 16 127599 127583 90.5982 68.5 0.00696948 0.0110324 2024-09-21T10:11:50.267 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:50.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:50.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:50.411 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:50.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:50.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:50.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:50.593 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:50.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:50.694 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:50.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:50.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:50.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:50.862 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:50.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:50.964 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:50.985 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:50.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:51.087 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.124 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:51.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:51.201 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 89 16 128646 128630 90.3154 65.4375 0.0140972 0.0110662 2024-09-21T10:11:51.225 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:51.261 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:51.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:51.363 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:51.399 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:51.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:51.501 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:51.534 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:51.636 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:51.674 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:51.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:51.776 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:51 smithi045 ceph-mon[95976]: pgmap v588: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-21T10:11:51.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:51.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:51.937 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:51 smithi032 ceph-mon[101299]: pgmap v588: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-21T10:11:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:51 smithi032 ceph-mon[105748]: pgmap v588: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-21T10:11:52.039 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.071 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:52.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:52.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:52.201 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total time run: 90.0062 2024-09-21T10:11:52.201 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total writes made: 129620 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Write size: 65536 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object size: 65536 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Bandwidth (MB/sec): 90.0077 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Bandwidth: 36.6808 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max bandwidth (MB/sec): 232.688 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min bandwidth (MB/sec): 59 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average IOPS: 1440 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev IOPS: 586.892 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max IOPS: 3723 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min IOPS: 944 2024-09-21T10:11:52.202 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average Latency(s): 0.011106 2024-09-21T10:11:52.203 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Latency(s): 0.00883874 2024-09-21T10:11:52.203 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max latency(s): 0.234546 2024-09-21T10:11:52.203 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min latency(s): 0.00124686 2024-09-21T10:11:52.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.204 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:52.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:52.206 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:11:52.305 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:52.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:52.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:52.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:52.579 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:52.618 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:52.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:52.719 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.825 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:52.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:52.927 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:52.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.965 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:52.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:53.066 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:53.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:53.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:53.201 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.238 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:53.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:53.339 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:53.374 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:53.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:53.522 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:53.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:53.624 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:53.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:53.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:53 smithi032 ceph-mon[101299]: pgmap v589: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:53 smithi032 ceph-mon[105748]: pgmap v589: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:53.763 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:53.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:53 smithi045 ceph-mon[95976]: pgmap v589: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:11:53.793 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:53.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:53.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:53.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:53.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:54.050 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:54.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:54.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:54.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.246 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:54.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:54.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:54.396 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:54.498 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:54.528 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:54.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:54.630 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:54.670 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:54.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:54.709 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:54.716 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:54.773 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.809 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:54.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:54.910 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.954 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:54.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:55.055 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:55.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:55.189 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:55.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:55.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:55.342 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:55.394 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:55.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.553 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:55.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:55.650 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:55 smithi045 ceph-mon[95976]: pgmap v590: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 85 B/s rd, 58 MiB/s wr, 938 op/s 2024-09-21T10:11:55.650 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:11:55.654 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.690 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:55.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:55.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:55 smithi032 ceph-mon[101299]: pgmap v590: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 85 B/s rd, 58 MiB/s wr, 938 op/s 2024-09-21T10:11:55.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:11:55.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:55 smithi032 ceph-mon[105748]: pgmap v590: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 85 B/s rd, 58 MiB/s wr, 938 op/s 2024-09-21T10:11:55.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:11:55.791 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:55.814 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:55.821 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.3 reset_purged_snaps_last 2024-09-21T10:11:55.917 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:55.952 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:55.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:56.053 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.095 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:56.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:56.196 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:56.224 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:56.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:56.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.364 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:56.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:56.465 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:56.496 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:56.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:56.597 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:56.644 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:56.745 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:56.786 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:56.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:56.887 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:56.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:56.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:57.028 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.067 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:57.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:57.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:57.203 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:57.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:57.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.339 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:57.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:57.440 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:57.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.485 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:57.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:57.509 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:11:57.585 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:57.619 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:57.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:57.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:57.754 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:57 smithi045 ceph-mon[95976]: pgmap v591: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.14k op/s 2024-09-21T10:11:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:57 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2783642953' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:57 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:57.857 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:57.892 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:57.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:57.905 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:57 smithi032 ceph-mon[105748]: pgmap v591: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.14k op/s 2024-09-21T10:11:57.905 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:57 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2783642953' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:57.905 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:57 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:57.905 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:57 smithi032 ceph-mon[101299]: pgmap v591: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.14k op/s 2024-09-21T10:11:57.905 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:57 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2783642953' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:57.905 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:57 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:57.994 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:58.028 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:58.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:58.130 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:58.191 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:58.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:58.293 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.335 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:58.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:58.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:58.477 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:58.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:58.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:58.621 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:58.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:58.723 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:58.758 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:58 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:58.759 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:58 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:58.759 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:58 smithi045 ceph-mon[95976]: osdmap e210: 8 total, 8 up, 7 in 2024-09-21T10:11:58.763 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:58.866 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:58.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:58.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:58 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:58.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:58 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:58.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:58 smithi032 ceph-mon[101299]: osdmap e210: 8 total, 8 up, 7 in 2024-09-21T10:11:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:58 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:58 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:58 smithi032 ceph-mon[105748]: osdmap e210: 8 total, 8 up, 7 in 2024-09-21T10:11:58.999 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:59.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:59.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:59.132 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:59.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:59.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:59.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:59.302 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:59.403 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:59.448 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:59.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:59.591 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:11:59.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:59.692 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:59.732 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:59.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:11:59 smithi045 ceph-mon[95976]: pgmap v593: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 46 MiB/s wr, 1.05k op/s 2024-09-21T10:11:59.834 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:59.875 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:11:59.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:59.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:11:59 smithi032 ceph-mon[101299]: pgmap v593: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 46 MiB/s wr, 1.05k op/s 2024-09-21T10:11:59.884 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:11:59 smithi032 ceph-mon[105748]: pgmap v593: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 46 MiB/s wr, 1.05k op/s 2024-09-21T10:11:59.884 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:11:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:11:59] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.51.0" 2024-09-21T10:11:59.976 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:00.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:00.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:00.115 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:00.150 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:00.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:00.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:00.288 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:00.390 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:00.427 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:00.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.568 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:00.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:00.669 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:00.706 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:00.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:00.807 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:00.853 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:00.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:00.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.995 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:00.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:00.999 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:12:00.999 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:12:00.999 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:12:00.999 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:12:00.999 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229196 5.15 5:ae9c4128:::benchmark_data_smithi032_140417_object95805:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:57.930746+0000", 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 183.00773955700001, 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.597960158, 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229196 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.000 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930746+0000", 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930746+0000", 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930748+0000", 2024-09-21T10:12:01.001 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2170000000000001e-06 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930756+0000", 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7819999999999994e-06 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930769+0000", 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3587000000000001e-05 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.002 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930780+0000", 2024-09-21T10:12:01.003 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0808999999999999e-05 2024-09-21T10:12:01.003 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.003 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931044+0000", 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00026428900000000002 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931056+0000", 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1551e-05 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.523492+0000", 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.592435815 2024-09-21T10:12:01.004 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.523606+0000", 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011398100000000001 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.523913+0000", 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00030749 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.005 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528333+0000", 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0044194539999999997 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528358+0000", 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5341999999999999e-05 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528402+0000", 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4100000000000001e-05 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.006 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528500+0000", 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7800000000000006e-05 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528557+0000", 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6910000000000002e-05 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528706+0000", 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014903099999999999 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.007 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229215 5.15 5:af0cbc59:::benchmark_data_smithi032_140417_object95824:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:57.930917+0000", 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 183.00756783599999, 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5981286350000001, 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229215 2024-09-21T10:12:01.008 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930917+0000", 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930917+0000", 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930918+0000", 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.23e-07 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930923+0000", 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4510000000000002e-06 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930934+0000", 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1936999999999999e-05 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.010 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930942+0000", 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.661e-06 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931156+0000", 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00021363999999999999 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931164+0000", 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3839999999999997e-06 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.011 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.525333+0000", 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.594168781 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.525447+0000", 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000113826 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.525603+0000", 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000156568 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528448+0000", 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0028446330000000001 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528455+0000", 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5129999999999999e-06 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528871+0000", 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00041531199999999998 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.013 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528947+0000", 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6681999999999995e-05 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528988+0000", 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0234999999999997e-05 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.529046+0000", 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8288999999999997e-05 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.014 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229301 5.15 5:ae7d68a4:::benchmark_data_smithi032_140417_object95910:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:57.930955+0000", 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 183.00753011399999, 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5983624189999999, 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.015 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229301 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930955+0000", 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930955+0000", 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.016 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930956+0000", 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6300000000000005e-07 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930960+0000", 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8940000000000003e-06 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930968+0000", 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2139999999999996e-06 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.017 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930974+0000", 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0379999999999996e-06 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931214+0000", 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024018999999999999 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931222+0000", 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5650000000000004e-06 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.018 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.527000+0000", 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5957785680000001 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.527119+0000", 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000119257 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.527352+0000", 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000232623 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.019 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528834+0000", 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014821890000000001 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.528905+0000", 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0862e-05 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.529100+0000", 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000194995 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.020 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.529157+0000", 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7144e-05 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.529203+0000", 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5575000000000003e-05 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.529317+0000", 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011474199999999999 2024-09-21T10:12:01.021 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229418 5.15 5:ab9a40a1:::benchmark_data_smithi032_140417_object96027:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:57.930983+0000", 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 183.00750256800001, 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.608598626, 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.022 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229418 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930983+0000", 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930983+0000", 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.023 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930983+0000", 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7300000000000001e-07 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930986+0000", 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5009999999999999e-06 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.930994+0000", 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1680000000000004e-06 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.024 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931004+0000", 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.028e-05 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931259+0000", 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00025508300000000002 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931267+0000", 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6210000000000004e-06 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.025 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.529490+0000", 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5982229110000001 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.529562+0000", 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2714999999999998e-05 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.529804+0000", 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024126900000000001 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.026 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539329+0000", 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0095254169999999996 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539400+0000", 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0840999999999995e-05 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539416+0000", 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6509000000000001e-05 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.027 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539442+0000", 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5984999999999999e-05 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539483+0000", 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0679999999999997e-05 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539581+0000", 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8172999999999997e-05 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.028 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229448 5.15 5:a9f00fe7:::benchmark_data_smithi032_140417_object96057:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:57.931085+0000", 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 183.00740051700001, 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.612235206, 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229448 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.029 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931085+0000", 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931085+0000", 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931086+0000", 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0440000000000001e-06 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.030 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931091+0000", 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9690000000000002e-06 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931124+0000", 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.3599000000000001e-05 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931134+0000", 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4339999999999998e-06 2024-09-21T10:12:01.031 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931320+0000", 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018598200000000001 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931329+0000", 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8759999999999994e-06 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.537257+0000", 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6059284310000002 2024-09-21T10:12:01.032 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.537367+0000", 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011018600000000001 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.537696+0000", 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00032925800000000002 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539683+0000", 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001986359 2024-09-21T10:12:01.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.542938+0000", 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0032557110000000001 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.542988+0000", 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9100000000000001e-05 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.543121+0000", 2024-09-21T10:12:01.034 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013330800000000001 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.543193+0000", 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2285000000000003e-05 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.543320+0000", 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012666399999999999 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.035 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229469 5.15 5:aa95ee2c:::benchmark_data_smithi032_140417_object96078:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:57.931151+0000", 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 183.007334235, 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6213226029999999, 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229469 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.036 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931151+0000", 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931151+0000", 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931152+0000", 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8100000000000002e-07 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.037 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931156+0000", 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2570000000000001e-06 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931164+0000", 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8189999999999992e-06 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931169+0000", 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4729999999999999e-06 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.038 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931480+0000", 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000311112 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931490+0000", 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.815e-06 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539705+0000", 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.608215127 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.039 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539785+0000", 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9554000000000004e-05 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.539979+0000", 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00019396999999999999 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.542952+0000", 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002973205 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.040 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.543054+0000", 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000102026 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.547482+0000", 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0044283619999999999 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552292+0000", 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0048095960000000002 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.041 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552360+0000", 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8083999999999994e-05 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552474+0000", 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000113522 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.042 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229480 5.15 5:a81aac3d:::benchmark_data_smithi032_140417_object96089:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:57.931233+0000", 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 183.00725171799999, 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6215024759999999, 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229480 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.043 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931233+0000", 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931233+0000", 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931234+0000", 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7000000000000003e-07 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.044 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931239+0000", 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5669999999999996e-06 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931253+0000", 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4467999999999999e-05 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931263+0000", 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5249999999999994e-06 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.045 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931513+0000", 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024959500000000003 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:57.931518+0000", 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3890000000000004e-06 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.541082+0000", 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.609563968 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.046 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.541183+0000", 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000101343 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.541393+0000", 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000209333 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.547210+0000", 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0058170670000000004 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.047 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.547535+0000", 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00032553500000000001 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.547583+0000", 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8130000000000002e-05 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552566+0000", 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.004983131 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.048 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552623+0000", 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6073000000000003e-05 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552736+0000", 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011338200000000001 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229514 5.15 5:ad614eb8:::benchmark_data_smithi032_140417_object96123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:58.014454+0000", 2024-09-21T10:12:01.049 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 182.92403126900001, 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5385501850000001, 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229514 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.014454+0000", 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.050 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.014454+0000", 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.014456+0000", 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8509999999999999e-06 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.014463+0000", 2024-09-21T10:12:01.051 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9280000000000004e-06 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.014478+0000", 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5244e-05 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.014486+0000", 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2600000000000005e-06 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.014560+0000", 2024-09-21T10:12:01.052 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3814999999999997e-05 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.014570+0000", 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0158e-05 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.543381+0000", 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5288104329999999 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.543468+0000", 2024-09-21T10:12:01.053 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7008000000000004e-05 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.543764+0000", 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029626899999999999 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.547661+0000", 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003896984 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.054 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.549958+0000", 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0022970669999999999 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552207+0000", 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0022488539999999998 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.055 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552840+0000", 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00063283999999999997 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552896+0000", 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6443999999999997e-05 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.553004+0000", 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010802999999999999 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.056 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229528 5.15 5:aa3eaa2e:::benchmark_data_smithi032_140417_object96137:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:58.028180+0000", 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 182.91030556300001, 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.525059398, 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229528 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.057 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.028180+0000", 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.028180+0000", 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.028181+0000", 2024-09-21T10:12:01.058 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7749999999999999e-06 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.028186+0000", 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6129999999999997e-06 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.028198+0000", 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2344000000000001e-05 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.028204+0000", 2024-09-21T10:12:01.059 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0290000000000002e-06 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.028264+0000", 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0019000000000003e-05 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.028276+0000", 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1663e-05 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.545231+0000", 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.516955327 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.545341+0000", 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000109677 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.545600+0000", 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00025914699999999999 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.061 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552108+0000", 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0065080709999999998 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552153+0000", 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5204000000000003e-05 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552518+0000", 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000364209 2024-09-21T10:12:01.062 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.553050+0000", 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00053271700000000004 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.553140+0000", 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0000999999999994e-05 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.553239+0000", 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8602000000000002e-05 2024-09-21T10:12:01.063 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229594 5.15 5:ac402dee:::benchmark_data_smithi032_140417_object96203:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:58.085806+0000", 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 182.85267955699999, 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.477798712, 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.064 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229594 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.085806+0000", 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.085806+0000", 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.065 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.085808+0000", 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3010000000000001e-06 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.085815+0000", 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6150000000000002e-06 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.085831+0000", 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6705999999999999e-05 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.066 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.085841+0000", 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6069999999999997e-06 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.085921+0000", 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0308000000000004e-05 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.085931+0000", 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7180000000000008e-06 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.067 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.547702+0000", 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.461771384 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.547807+0000", 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000104384 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.548099+0000", 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029215600000000001 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.068 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552166+0000", 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0040675310000000001 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552249+0000", 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2628999999999998e-05 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.563328+0000", 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011079383 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.069 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.563382+0000", 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3903000000000002e-05 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.563483+0000", 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000100828 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.563604+0000", 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000121259 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.070 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229696 5.15 5:a95a1648:::benchmark_data_smithi032_140417_object96305:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:58.192514+0000", 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 182.74597083399999, 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.371777662, 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229696 2024-09-21T10:12:01.071 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.192514+0000", 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.192514+0000", 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.192516+0000", 2024-09-21T10:12:01.072 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1239999999999999e-06 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.192521+0000", 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3800000000000004e-06 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.192534+0000", 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3545e-05 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.192542+0000", 2024-09-21T10:12:01.073 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1559999999999998e-06 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.192602+0000", 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0383000000000001e-05 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.192614+0000", 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2384e-05 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.549989+0000", 2024-09-21T10:12:01.074 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3573751569999999 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.550106+0000", 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000116619 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.550415+0000", 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00030931900000000001 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552530+0000", 2024-09-21T10:12:01.075 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0021150240000000001 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.552787+0000", 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00025694500000000001 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.564067+0000", 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011280091000000001 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.564113+0000", 2024-09-21T10:12:01.076 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5460000000000002e-05 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.564202+0000", 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8739999999999994e-05 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.564292+0000", 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0334999999999995e-05 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.077 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.078 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229709 5.15 5:a8857519:::benchmark_data_smithi032_140417_object96318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.078 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:58.206198+0000", 2024-09-21T10:12:01.078 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 182.73228715299999, 2024-09-21T10:12:01.078 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3641125510000001, 2024-09-21T10:12:01.078 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.078 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.078 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.078 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.078 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.079 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.079 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229709 2024-09-21T10:12:01.079 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.079 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.079 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.206198+0000", 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.206198+0000", 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.206199+0000", 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.091e-06 2024-09-21T10:12:01.080 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.206203+0000", 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0350000000000003e-06 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.206214+0000", 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1045e-05 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.206220+0000", 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6230000000000002e-06 2024-09-21T10:12:01.081 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.206264+0000", 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4150999999999998e-05 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.206270+0000", 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7429999999999999e-06 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.553269+0000", 2024-09-21T10:12:01.082 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3469995930000001 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.553377+0000", 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010793099999999999 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.553711+0000", 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00033400299999999998 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.083 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.563654+0000", 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0099426670000000005 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.569930+0000", 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0062761340000000001 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.569944+0000", 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4066999999999999e-05 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.084 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570093+0000", 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014873800000000001 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570183+0000", 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0268999999999999e-05 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570311+0000", 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012746099999999999 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.085 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229724 5.15 5:afa64470:::benchmark_data_smithi032_140417_object96333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:58.224219+0000", 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 182.714265834, 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.346546818, 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229724 2024-09-21T10:12:01.086 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.224219+0000", 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.224219+0000", 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.224221+0000", 2024-09-21T10:12:01.087 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4410000000000001e-06 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.224225+0000", 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3959999999999999e-06 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.224239+0000", 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.376e-05 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.224247+0000", 2024-09-21T10:12:01.088 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2740000000000007e-06 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.224304+0000", 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7247000000000003e-05 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.224314+0000", 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1479999999999995e-06 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.555432+0000", 2024-09-21T10:12:01.089 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.331118265 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.555547+0000", 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011466100000000001 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.555878+0000", 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00033122300000000001 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.564340+0000", 2024-09-21T10:12:01.090 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0084621409999999994 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.569950+0000", 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0056099970000000002 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570373+0000", 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00042277100000000001 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570541+0000", 2024-09-21T10:12:01.091 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016874699999999999 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570663+0000", 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000121898 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570766+0000", 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000102849 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.092 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229773 5.15 5:a9b4e206:::benchmark_data_smithi032_140417_object96382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:58.289008+0000", 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 182.64947759899999, 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2822879979999999, 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229773 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.093 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.289008+0000", 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.289008+0000", 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.289009+0000", 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3769999999999999e-06 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.094 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.289014+0000", 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2989999999999996e-06 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.289026+0000", 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.167e-05 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.289032+0000", 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8050000000000003e-06 2024-09-21T10:12:01.095 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.289087+0000", 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5053000000000003e-05 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.289094+0000", 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.576e-06 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.557591+0000", 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2684967540000001 2024-09-21T10:12:01.096 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.557711+0000", 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011995299999999999 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.558094+0000", 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00038319 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.569961+0000", 2024-09-21T10:12:01.097 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011866984000000001 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.569999+0000", 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.7481999999999998e-05 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570808+0000", 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00080929799999999996 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570927+0000", 2024-09-21T10:12:01.098 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000119059 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.571067+0000", 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013947499999999999 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.571296+0000", 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00022902299999999999 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.099 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:229785 5.15 5:abef2239:::benchmark_data_smithi032_140417_object96394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:58.310059+0000", 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 182.628425892, 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.261556699, 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 229785 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.100 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.310059+0000", 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.310059+0000", 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.310061+0000", 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2079999999999999e-06 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.101 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.310068+0000", 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0959999999999997e-06 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.310084+0000", 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6313000000000001e-05 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.310093+0000", 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5990000000000008e-06 2024-09-21T10:12:01.102 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.310162+0000", 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8759999999999999e-05 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:58.310174+0000", 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1841000000000001e-05 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.560247+0000", 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.250073086 2024-09-21T10:12:01.103 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.560366+0000", 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000119134 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4,5", 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.560669+0000", 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00030274399999999997 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570052+0000", 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0093827619999999994 2024-09-21T10:12:01.104 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.570424+0000", 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00037169299999999998 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.571342+0000", 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00091803999999999996 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.571381+0000", 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9032999999999997e-05 2024-09-21T10:12:01.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.571512+0000", 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013119100000000001 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:59.571616+0000", 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000104199 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.106 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:239836 5.3 5:c5e8b8e4:::benchmark_data_smithi032_140417_object106445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:12.530977+0000", 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 168.40750817099999, 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.108278109, 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 239836 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.107 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.530977+0000", 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.530977+0000", 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.530979+0000", 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0229999999999999e-06 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.108 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.530984+0000", 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3419999999999998e-06 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.530997+0000", 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2568000000000001e-05 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.531004+0000", 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3270000000000003e-06 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.109 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.531065+0000", 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0550999999999997e-05 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.531074+0000", 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9029999999999993e-06 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.635531+0000", 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1044570199999999 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.110 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.635646+0000", 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000115015 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.636137+0000", 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00049076100000000004 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.638654+0000", 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0025174730000000001 2024-09-21T10:12:01.111 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.639095+0000", 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00044083399999999997 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.639145+0000", 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0686999999999998e-05 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.639206+0000", 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0207999999999997e-05 2024-09-21T10:12:01.112 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.639255+0000", 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9397000000000002e-05 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:239844 5.3 5:c677c9be:::benchmark_data_smithi032_140417_object106453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:12.544440+0000", 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 168.394044708, 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1061434880000001, 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.113 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 239844 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.544440+0000", 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.114 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.544440+0000", 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.544442+0000", 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.03e-06 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.544447+0000", 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9989999999999999e-06 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.115 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.544460+0000", 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2860999999999999e-05 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.544467+0000", 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0520000000000004e-06 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.544517+0000", 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9978999999999997e-05 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.116 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.544526+0000", 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3699999999999995e-06 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.639297+0000", 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0947709880000001 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.639420+0000", 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012326500000000001 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.117 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.639858+0000", 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00043794299999999999 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.645562+0000", 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0057045009999999998 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.645616+0000", 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3587e-05 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.118 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650511+0000", 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0048946980000000003 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650542+0000", 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0966000000000001e-05 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650584+0000", 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2249000000000002e-05 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.119 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:239909 5.3 5:c7a62a30:::benchmark_data_smithi032_140417_object106518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:12.610175+0000", 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 168.32830983100001, 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0405646639999999, 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 239909 2024-09-21T10:12:01.120 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.610175+0000", 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.610175+0000", 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.610177+0000", 2024-09-21T10:12:01.121 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7150000000000001e-06 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.610183+0000", 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.610196+0000", 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3074000000000001e-05 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.610204+0000", 2024-09-21T10:12:01.122 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8869999999999992e-06 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.610852+0000", 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00064821500000000005 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.610862+0000", 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4460000000000007e-06 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.642326+0000", 2024-09-21T10:12:01.123 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0314642279999999 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.642454+0000", 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000128675 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.642855+0000", 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00040002800000000003 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.124 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650502+0000", 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.007647755 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650644+0000", 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000142137 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650690+0000", 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.532e-05 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650718+0000", 2024-09-21T10:12:01.125 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.8547999999999998e-05 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650740+0000", 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1736e-05 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:239916 5.3 5:c0e681b3:::benchmark_data_smithi032_140417_object106525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:12.613493+0000", 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 168.32499264099999, 2024-09-21T10:12:01.126 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.037345481, 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 239916 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.613493+0000", 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.127 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.613493+0000", 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.613494+0000", 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9010000000000001e-06 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.613499+0000", 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.955e-06 2024-09-21T10:12:01.128 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.613517+0000", 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8105e-05 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.613526+0000", 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2749999999999995e-06 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.613661+0000", 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000135229 2024-09-21T10:12:01.129 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.613670+0000", 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3200000000000006e-06 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.645645+0000", 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0319742139999999 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.645748+0000", 2024-09-21T10:12:01.130 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010357900000000001 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.646164+0000", 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00041618699999999999 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650682+0000", 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0045178290000000001 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.131 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650744+0000", 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2374999999999998e-05 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650780+0000", 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5747000000000002e-05 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650814+0000", 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.3745999999999998e-05 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.132 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650838+0000", 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4019000000000002e-05 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:239942 5.3 5:c6bb4a97:::benchmark_data_smithi032_140417_object106551:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:12.638586+0000", 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 168.29989958100001, 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.019606655, 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 239942 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.638586+0000", 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.638586+0000", 2024-09-21T10:12:01.134 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.638588+0000", 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0269999999999998e-06 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.638594+0000", 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7279999999999998e-06 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.638615+0000", 2024-09-21T10:12:01.135 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0783000000000001e-05 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.638625+0000", 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0012e-05 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.638698+0000", 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3242000000000001e-05 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:12.638712+0000", 2024-09-21T10:12:01.136 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3675e-05 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.648318+0000", 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0096055900000001 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.648488+0000", 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016998299999999999 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.648911+0000", 2024-09-21T10:12:01.137 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00042296599999999999 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650748+0000", 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0018374159999999999 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.650893+0000", 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000144764 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.658072+0000", 2024-09-21T10:12:01.138 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0071794229999999999 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.658137+0000", 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4777999999999995e-05 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:13.658192+0000", 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5268e-05 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.139 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:12:01.140 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:12:01.140 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:12:01.140 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:01.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.176 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:01.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:01.277 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:01.323 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:01.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:01.424 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:01.463 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:01.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:01.565 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:01.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:01.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:01.697 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:01.700 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:01.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:01.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:01.869 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:01.898 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:01 smithi032 ceph-mon[101299]: pgmap v594: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 1.05k op/s 2024-09-21T10:12:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:01 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:01 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:01 smithi032 ceph-mon[105748]: pgmap v594: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 1.05k op/s 2024-09-21T10:12:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:01 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:01 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:02.000 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:02.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:02.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:02.142 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:02.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:02.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:02.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:01 smithi045 ceph-mon[95976]: pgmap v594: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 1.05k op/s 2024-09-21T10:12:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:01 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:01 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:02.319 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:02.421 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:02.450 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:02.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:02.589 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:02.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:02.691 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:02.728 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:02.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:02.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:02.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:12:02.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:02.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:02.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:03.024 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:03.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:03.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:03.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.164 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:03.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:03.265 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.303 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:03.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:03.404 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:03.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:03.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:03.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:03.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:03.705 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:03.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:03.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:12:03.840 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:03.877 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:03.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:03.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:03 smithi032 ceph-mon[101299]: pgmap v595: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 1.04k op/s 2024-09-21T10:12:03.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:03.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:03.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:03 smithi032 ceph-mon[105748]: pgmap v595: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 1.04k op/s 2024-09-21T10:12:03.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:03.960 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:03.979 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:04.044 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:04.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:04.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:04.185 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:04.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:03 smithi045 ceph-mon[95976]: pgmap v595: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 1.04k op/s 2024-09-21T10:12:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:04.286 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:04.318 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:04.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:04.419 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:04.454 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:04.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:04.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.592 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:04.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:04.693 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:04.733 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:04.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:04.835 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:04.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:04.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:04.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:05.009 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:05.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:05.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:05.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:05.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:05.249 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:05.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.288 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:05.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:05.389 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:05.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:05.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:05.531 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:05.565 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:05.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:05.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:05.704 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:05.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:05.806 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:05.849 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:05.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:05.951 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:05.962 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:12:05.963 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:05.991 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:06.001 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:12:06.001 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:06.041 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:12:06.041 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:06.075 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:12:06.075 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:06.093 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:06.111 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:12:06.111 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:06.122 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:06.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:06.141 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:12:06.141 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:06.180 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:12:06.180 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:06.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:05 smithi032 ceph-mon[101299]: pgmap v596: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 11 MiB/s wr, 1.04k op/s 2024-09-21T10:12:06.180 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:05 smithi032 ceph-mon[105748]: pgmap v596: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 11 MiB/s wr, 1.04k op/s 2024-09-21T10:12:06.213 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:12:06.213 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:06.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:06.283 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:06.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:05 smithi045 ceph-mon[95976]: pgmap v596: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 11 MiB/s wr, 1.04k op/s 2024-09-21T10:12:06.385 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:06.419 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:06.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:06.521 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:06.556 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:06.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:06.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:06.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:06.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:06.828 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:06.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:06.930 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:06.966 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:06.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:07.068 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:07.105 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:07.207 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:07.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/372762150' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:07.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/372762150' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:07.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.247 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:07.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/372762150' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:07.348 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:07.389 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:07.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:07.488 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.4 reset_purged_snaps_last 2024-09-21T10:12:07.491 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:07.540 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:07.542 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.566 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:07.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:07.667 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:07.711 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:07.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:07.817 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.849 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:07.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:07.950 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:07.998 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:08.101 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:08.136 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:08.190 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:07 smithi032 ceph-mon[101299]: pgmap v597: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 806 KiB/s wr, 1.03k op/s 2024-09-21T10:12:08.190 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:07 smithi032 ceph-mon[105748]: pgmap v597: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 806 KiB/s wr, 1.03k op/s 2024-09-21T10:12:08.238 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:08.271 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:08.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:07 smithi045 ceph-mon[95976]: pgmap v597: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 806 KiB/s wr, 1.03k op/s 2024-09-21T10:12:08.373 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:08.416 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:08.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:08.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:08.563 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:08.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:08.665 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:08.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:08.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:08.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:08.842 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:08.944 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:08.983 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:09.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:09.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:09.231 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:09.273 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:09.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:09.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:09.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.413 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:09.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:09.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:09.553 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:09.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:09.654 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:09.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:09.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:09.789 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:09.823 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:09.912 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:12:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:12:09] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.51.0" 2024-09-21T10:12:09.924 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:09.957 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:09.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:10.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:12:10.059 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:10.104 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:10.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:09 smithi032 ceph-mon[105748]: pgmap v598: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 973 op/s 2024-09-21T10:12:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:09 smithi032 ceph-mon[101299]: pgmap v598: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 973 op/s 2024-09-21T10:12:10.239 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:10.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:09 smithi045 ceph-mon[95976]: pgmap v598: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 973 op/s 2024-09-21T10:12:10.341 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:10.377 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:10.478 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:10.514 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:10.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:10.651 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:10.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:10.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:10.786 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:10.888 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:10.923 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:10.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:11.026 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:11.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.064 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:11.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:11.165 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.199 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:11.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:11.300 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:11.331 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:11.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:11.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:11.469 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:11.559 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:11.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:11.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:11.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:11.742 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:11.779 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:11.881 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:11.915 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:12.017 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:12.048 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:12.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:12.076 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:11 smithi032 ceph-mon[105748]: pgmap v599: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-21T10:12:12.076 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:11 smithi032 ceph-mon[101299]: pgmap v599: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-21T10:12:12.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:12.190 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:12.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:11 smithi045 ceph-mon[95976]: pgmap v599: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-21T10:12:12.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:12.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:12.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:12.414 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:12.431 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:12.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.485 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:12.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:12.586 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:12.630 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:12.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:12.732 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:12.773 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:12.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:12.874 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:12.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:12.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:12.924 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:12:13.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:13.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.070 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:13.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:13.171 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:13.205 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:12 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1085642631' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:13.205 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:12 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:13.205 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:12 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1085642631' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:13.205 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:12 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:13.213 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:13.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:13.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:12 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1085642631' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:13.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:12 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:13.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:13.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.351 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:13.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:13.426 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:13.426 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:13.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:13.503 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:13.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:13.604 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:13.639 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:13.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:13.741 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:13.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.779 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:13.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:13.880 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:13.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.917 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:13.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:14.018 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:14.052 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:14.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:14.154 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:14.186 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:13 smithi045 ceph-mon[95976]: pgmap v600: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:12:14.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:13 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:14.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:13 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:14.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:13 smithi045 ceph-mon[95976]: osdmap e211: 8 total, 8 up, 7 in 2024-09-21T10:12:14.190 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:13 smithi032 ceph-mon[105748]: pgmap v600: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:12:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:13 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:13 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:13 smithi032 ceph-mon[105748]: osdmap e211: 8 total, 8 up, 7 in 2024-09-21T10:12:14.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:13 smithi032 ceph-mon[101299]: pgmap v600: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:12:14.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:13 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:14.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:13 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:14.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:13 smithi032 ceph-mon[101299]: osdmap e211: 8 total, 8 up, 7 in 2024-09-21T10:12:14.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:14.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.325 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:14.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:14.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:14.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.461 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:14.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:14.562 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:14.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:14.706 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:14.757 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:14.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:14.858 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:14.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.905 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:14.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:15.006 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:15.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.052 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:15.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:15.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:15.193 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:15.294 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:15.327 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:15.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:15.428 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:15.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.463 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:15.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:15.564 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:15.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.611 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:15.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:15.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:15.743 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:15.845 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:15.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.884 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:15.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:15.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:16.018 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:16.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:16.120 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:16.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.147 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:16.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:16.194 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:15 smithi045 ceph-mon[95976]: pgmap v602: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-21T10:12:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:15 smithi032 ceph-mon[105748]: pgmap v602: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-21T10:12:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:15 smithi032 ceph-mon[101299]: pgmap v602: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-21T10:12:16.247 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:16.281 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:16.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:16.383 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.423 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:16.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:16.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:16.559 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:16.664 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.710 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:16.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:16.811 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:16.844 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:16.946 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:16.979 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:17.081 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:17.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:17.250 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:17.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:17.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:17.384 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:17.419 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:17.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:17.521 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:17.567 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:17.669 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:17.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:17.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:17.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:17.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.857 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:17.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:17.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:17.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.994 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:17.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:18.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:18.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:18.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:18.134 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:17 smithi032 ceph-mon[105748]: pgmap v603: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-21T10:12:18.135 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:17 smithi032 ceph-mon[101299]: pgmap v603: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-21T10:12:18.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:18.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.264 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:18.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:17 smithi045 ceph-mon[95976]: pgmap v603: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-21T10:12:18.365 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:18.403 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:18.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:18.505 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:18.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.548 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:18.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:18.589 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:12:18.649 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:18.688 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:18.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:18.790 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:18.823 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:18.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:18.925 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:18.965 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:19.067 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:19.093 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:19.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:19.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:19.229 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:19.331 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:19.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.373 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:19.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:19.473 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:19.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.512 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:19.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:19.613 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:19.649 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:19.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:19.751 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:19.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:19.896 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:19.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:19.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:19.950 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:12:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:12:19] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.51.0" 2024-09-21T10:12:20.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:20.064 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:20.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:20.203 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:20.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:19 smithi032 ceph-mon[105748]: pgmap v604: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-21T10:12:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:19 smithi032 ceph-mon[101299]: pgmap v604: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-21T10:12:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:19 smithi045 ceph-mon[95976]: pgmap v604: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-21T10:12:20.290 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:12:20.305 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:20.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:20.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:20.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:20.480 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:20.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:20.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.619 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:20.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:20.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:20.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:20.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:20.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:20.893 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:20.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:20.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:21.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.032 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:21.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:21.133 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:21.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:21.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:21.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:21.302 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:21.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:21.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:21.440 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:21.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:21.542 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:21.575 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:21.677 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:21.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:21.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:21.811 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:21.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.847 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:21.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:21.948 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.985 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:21.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:22.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:22.124 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:22.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:22.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:21 smithi032 ceph-mon[105748]: pgmap v605: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:21 smithi032 ceph-mon[101299]: pgmap v605: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:22.226 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:22.263 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:22.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:21 smithi045 ceph-mon[95976]: pgmap v605: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:22.365 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:22.402 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:22.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:22.470 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.5 reset_purged_snaps_last 2024-09-21T10:12:22.499 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:22.503 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:22.514 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:22.531 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:22.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:22.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:22.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:22.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:22.774 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:22.826 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:22.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.964 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:22.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:23.065 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:23.097 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:23.199 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:23.235 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:23.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:23.337 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:23.374 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:23.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:23.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:23.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.548 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:23.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:23.649 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:23.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.677 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:23.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:23.778 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:23.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.815 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:23.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:23.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:23.950 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:23.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.966 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:23.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:24.067 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:24.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.098 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:24.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:24.199 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:24.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:23 smithi032 ceph-mon[105748]: pgmap v606: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:24.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:24.217 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:23 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:24.217 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:23 smithi032 ceph-mon[101299]: pgmap v606: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:24.217 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:24.217 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:23 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:24.231 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:24.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:23 smithi045 ceph-mon[95976]: pgmap v606: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:23 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:23 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:24.333 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:24.371 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:24.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:24.473 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:24.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.512 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:24.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:24.613 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:24.649 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:24.751 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:24.789 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:24.891 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:24.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:24.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:25.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:25.065 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:25.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:25.167 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:25.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.201 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:25.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:24 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:12:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:24 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:12:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:24 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:24 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:12:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:24 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:12:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:24 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:24 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:12:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:24 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:12:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:24 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:25.302 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:25.334 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:25.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:25.469 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:25.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:25.571 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:25.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:25.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:25.710 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:25.740 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:25.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:25.841 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:25.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:25.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:25.983 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:26.022 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:26.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:26.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:26.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:26.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:26.231 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:12:26.232 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:26.232 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:25 smithi032 ceph-mon[101299]: pgmap v607: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 997 op/s 2024-09-21T10:12:26.232 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:25 smithi032 ceph-mon[105748]: pgmap v607: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 997 op/s 2024-09-21T10:12:26.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:26.285 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:12:26.285 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:26.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:25 smithi045 ceph-mon[95976]: pgmap v607: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 997 op/s 2024-09-21T10:12:26.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.314 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:26.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:26.318 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:12:26.318 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:26.354 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:12:26.354 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:26.391 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:12:26.392 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:26.415 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:26.423 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:12:26.423 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:26.450 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:26.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:26.456 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:12:26.456 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:26.488 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:12:26.488 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:26.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:26.590 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:26.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:26.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:26.734 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:26.836 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:26.873 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:26.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:26.975 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:26.982 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:12:27.007 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:27.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:27.109 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:27.146 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:27.248 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:27.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:27.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4281094646' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3898322841' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:26 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:27.343 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4281094646' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:27.343 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3898322841' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:27.343 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:26 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:27.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4281094646' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:27.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3898322841' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:27.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:26 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:27.385 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:27.426 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:27.529 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:27.572 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:27.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:27.674 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:27.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:27.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:27.811 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:27.843 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:27.945 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:27.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.981 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:27.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:28.082 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:28.112 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:28.215 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:28.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:27 smithi032 ceph-mon[101299]: pgmap v608: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-21T10:12:28.230 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:27 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:28.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:27 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:12:28.231 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:27 smithi032 ceph-mon[101299]: osdmap e212: 8 total, 8 up, 7 in 2024-09-21T10:12:28.231 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:27 smithi032 ceph-mon[105748]: pgmap v608: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-21T10:12:28.231 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:27 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:28.231 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:27 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:12:28.231 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:27 smithi032 ceph-mon[105748]: osdmap e212: 8 total, 8 up, 7 in 2024-09-21T10:12:28.251 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:27 smithi045 ceph-mon[95976]: pgmap v608: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-21T10:12:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:27 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:27 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:12:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:27 smithi045 ceph-mon[95976]: osdmap e212: 8 total, 8 up, 7 in 2024-09-21T10:12:28.352 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:28.384 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:28.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:28.486 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:28.523 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:28.625 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:28.663 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:28.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:28.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:28.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:28.901 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:28.935 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:28.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:29.037 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:29.075 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:29.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:29.176 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:29.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.212 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:29.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:29.313 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:29.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.352 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:29.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:29.453 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:29.465 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:12:29.465 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:12:29.465 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:12:29.465 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:12:29.465 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:12:29.487 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:29.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:29.589 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:29.628 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:29.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:29.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:12:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:12:29] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.51.0" 2024-09-21T10:12:29.730 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:29.770 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:29.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:29.872 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:29.905 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:30.007 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:30.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:12:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.061 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:30.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:30.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:30.197 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:30.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:29 smithi032 ceph-mon[101299]: pgmap v610: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:29 smithi032 ceph-mon[105748]: pgmap v610: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:29 smithi045 ceph-mon[95976]: pgmap v610: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:30.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:30.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:30.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:30.443 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:30.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:12:30.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:30.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:30.573 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:30.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:30.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:30.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:30.761 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:30.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:12:30.863 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:30.898 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:30.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:30.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:31.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.037 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:31.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:31.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:31.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:31.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:31.276 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:31.301 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:31.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:31.402 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:31.414 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:31.472 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:31.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:31.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.612 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:31.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:31.691 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:31.713 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:31.738 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:31.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:31.840 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:31.874 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:31.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:31.975 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:32.029 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:32.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:32.054 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:31 smithi032 ceph-mon[101299]: pgmap v611: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:32.055 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:31 smithi032 ceph-mon[105748]: pgmap v611: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:32.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:32.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:32.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:32.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:32.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:31 smithi045 ceph-mon[95976]: pgmap v611: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:32.309 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:32.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:32.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.452 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:32.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:32.515 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.6 reset_purged_snaps_last 2024-09-21T10:12:32.553 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.596 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:32.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:32.697 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:32.732 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:32.833 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:32.864 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:32.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:32.966 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:33.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.001 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:33.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:33.102 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:33.136 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:33.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:33.238 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:33.274 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:33.376 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:33.412 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:33.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:33.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:33.551 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:33.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:33.649 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:33.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:33.689 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:33.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:33.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:33.826 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:33.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:33.928 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:33.964 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:34.066 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:34.099 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:34.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:34.140 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:33 smithi032 ceph-mon[105748]: pgmap v612: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:34.140 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:34.140 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:33 smithi032 ceph-mon[101299]: pgmap v612: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:34.140 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:34.202 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:34.243 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:34.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:33 smithi045 ceph-mon[95976]: pgmap v612: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:34.345 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:34.377 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:34.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:34.479 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:34.513 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:34.615 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:34.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:34.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:34.758 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:34.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.804 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:34.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:34.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:34.940 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:34.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:35.043 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:35.081 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:35.182 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:35.221 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:35.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:35.323 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:35.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:35.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:35.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:35.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:35.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:35.613 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:35.653 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:35.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:35.755 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:35.788 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:35.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:35.889 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.927 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:35.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:36.028 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:36.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.063 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:36.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:36.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:36.199 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:35 smithi045 ceph-mon[95976]: pgmap v613: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:36.300 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:36.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.341 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:36.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:36.396 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:35 smithi032 ceph-mon[101299]: pgmap v613: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:36.396 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:35 smithi032 ceph-mon[105748]: pgmap v613: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:36.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:36.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:36.577 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:36.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.624 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:36.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:36.714 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:36.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:36.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:36.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:36.794 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:12:36.794 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:36.823 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:12:36.823 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:36.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:36.855 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:12:36.855 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:36.889 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:12:36.889 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:36.895 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:36.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:36.922 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:12:36.923 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:36.954 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:12:36.955 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:36.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:36.997 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:12:36.998 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:37.051 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:12:37.051 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:37.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.070 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:37.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:37.171 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:37 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/937362481' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:37 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/937362481' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:37.214 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:37.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:37 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/937362481' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:37.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:37.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:37.456 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:37.489 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:37.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:37.591 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:37.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:37.723 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:37.756 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:37.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:37.858 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:37.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:37.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:37.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:38.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:38.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:38.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:38.168 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:38.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:38.216 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:38 smithi032 ceph-mon[105748]: pgmap v614: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-21T10:12:38.216 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:38 smithi032 ceph-mon[101299]: pgmap v614: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-21T10:12:38.262 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:38.263 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:38.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:38 smithi045 ceph-mon[95976]: pgmap v614: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-21T10:12:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.309 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:38.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:38.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:38.445 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:38.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:38.546 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.581 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:38.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:38.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:38.717 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:38.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:38.818 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.855 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:38.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:38.955 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:38.986 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:38.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:39.088 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:39.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.129 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:39.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:39.230 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:39.263 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:39.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:39.365 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.401 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:39.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:39.502 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:39.532 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:39.612 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:12:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:12:39] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.51.0" 2024-09-21T10:12:39.633 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:39.665 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:39.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:39.767 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:39.808 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:39.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:39.910 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:39.943 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:39.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:40.045 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.083 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:40.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:40.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:40 smithi032 ceph-mon[105748]: pgmap v615: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 922 op/s 2024-09-21T10:12:40.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:40 smithi032 ceph-mon[101299]: pgmap v615: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 922 op/s 2024-09-21T10:12:40.220 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:40.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:40 smithi045 ceph-mon[95976]: pgmap v615: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 922 op/s 2024-09-21T10:12:40.321 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:40.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:40.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:40.458 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:40.494 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:40.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:40.595 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:40.630 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:40.732 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:40.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:40.867 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:40.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:40.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:40.998 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:41.034 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:41.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:41.039 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:12:41.136 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:41.177 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:41.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:41.279 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:41 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/306302997' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:41 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:41.311 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:41.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:41.413 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:41.450 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:41.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:41 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/306302997' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:41.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:41 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:41 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/306302997' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:41 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:41.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:41.584 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:12:41.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.611 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:41.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:41.713 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:41.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:41.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:41.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:41.900 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:41.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:42.001 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:42.042 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:42.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:42.144 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:42.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.199 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:42.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:42.274 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[101299]: pgmap v616: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-21T10:12:42.274 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:12:42.274 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:12:42.274 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:42.274 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[101299]: osdmap e213: 8 total, 8 up, 7 in 2024-09-21T10:12:42.274 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[105748]: pgmap v616: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-21T10:12:42.274 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:12:42.275 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:12:42.275 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:42.275 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:42 smithi032 ceph-mon[105748]: osdmap e213: 8 total, 8 up, 7 in 2024-09-21T10:12:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:42 smithi045 ceph-mon[95976]: pgmap v616: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-21T10:12:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:42 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:12:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:42 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:12:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:42 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:42 smithi045 ceph-mon[95976]: osdmap e213: 8 total, 8 up, 7 in 2024-09-21T10:12:42.300 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:42.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:42.448 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:42.483 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:42.585 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:42.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.623 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:42.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:42.724 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:42.749 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:42.762 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:42.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:42.863 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:42.910 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:43.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:43.049 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:43.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:43.151 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:43.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.185 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:43.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:43.208 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:12:43.287 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:43.319 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:43.421 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:43.443 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:43.545 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:43.578 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:43.680 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:43.713 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:43.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:43.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.850 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:43.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:43.951 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:43.977 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:43.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:44.078 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:44.108 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:12:44.118 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:44.220 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:44.258 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:44.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:44 smithi045 ceph-mon[95976]: pgmap v618: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:44.360 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:44.384 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:44.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:44 smithi032 ceph-mon[101299]: pgmap v618: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:44 smithi032 ceph-mon[105748]: pgmap v618: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:44.486 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:44.528 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:44.629 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:44.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:44.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:44.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:44.801 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:44.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:44.903 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.941 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:44.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:45.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:45.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.079 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:45.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:45.180 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.224 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:45.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:45.325 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:45.362 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:45.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:45.464 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:45.502 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:45.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:45.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:45.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.645 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:45.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:45.746 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:45.783 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:45.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:45.885 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:45.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:46.018 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:46.056 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:46.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:46.158 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:46.197 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:46.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:46 smithi045 ceph-mon[95976]: pgmap v619: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:46.298 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:46.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:46.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:46.439 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:46 smithi032 ceph-mon[105748]: pgmap v619: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:46 smithi032 ceph-mon[101299]: pgmap v619: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:46.479 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:46.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:46.580 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:46.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:46.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:46.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.745 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:46.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:46.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:46.868 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:46.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:46.970 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:47.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.009 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:47.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:47.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:47.139 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:47.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:47.241 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:47.270 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:47.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:47.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:47.408 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:47.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:47.461 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:47.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:47.546 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:47.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:47.647 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:47.689 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:47.791 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:47.831 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:47.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:47.933 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.971 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:47.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:48.071 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:48.107 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:48.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:48.203 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:48 smithi032 ceph-mon[101299]: pgmap v620: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:48.203 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:48 smithi032 ceph-mon[105748]: pgmap v620: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:48.209 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:48.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:48.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:48.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.7 reset_purged_snaps_last 2024-09-21T10:12:48.342 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:48.378 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:48.479 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:48.526 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:48.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:48.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:48 smithi045 ceph-mon[95976]: pgmap v620: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:48.627 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:48.662 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:48.763 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:48.799 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:48.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:48.809 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:48.872 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:48.873 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:48.901 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.944 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:48.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:49.045 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:49.081 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:49.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:49.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:49.224 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:49.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:49.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:49.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:49.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:49.463 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:49.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:49.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:49.599 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:49.635 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:49.719 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:49 smithi032 ceph-mon[105748]: pgmap v621: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:49.719 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:49.719 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:49.720 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:12:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:12:49] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.51.0" 2024-09-21T10:12:49.720 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:49 smithi032 ceph-mon[101299]: pgmap v621: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:49.720 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:49.720 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:49.736 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:49.773 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:49.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:49 smithi045 ceph-mon[95976]: pgmap v621: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:12:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:12:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:49.875 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:49.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.909 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:49.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:50.010 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:50.043 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:50.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:50.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:50.184 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:50.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:50.285 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:50.319 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:50.420 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:50.458 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:50.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:50.486 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:12:50.561 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:50.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:50.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:50.699 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:50.735 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:50.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:50.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.867 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:50.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:50.917 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:50.967 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:50.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:50.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:51.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:51.132 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:51.234 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:51.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.266 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:51.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:51.367 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:51.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.404 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:51.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:51.505 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:51.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.539 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:51.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:51.640 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:51.674 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:51.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:51.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:51 smithi032 ceph-mon[105748]: pgmap v622: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:12:51.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:51 smithi032 ceph-mon[101299]: pgmap v622: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:12:51.776 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:51 smithi045 ceph-mon[95976]: pgmap v622: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:12:51.808 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:51.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:51.910 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:51.940 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:51.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:52.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:52.082 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:52.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:52.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:52.226 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:52.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:52.328 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:52.377 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:52.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:52.478 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:52.513 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:52.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:52.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.653 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:52.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:52.754 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.789 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:52.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:52.890 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.926 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:52.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:53.027 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:53.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.065 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:53.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:53.166 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:53.197 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:53.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:53.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:53.336 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:53.437 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:53.450 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:53.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:53.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:53.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:53.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.606 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:53.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:53.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:53.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:53 smithi032 ceph-mon[105748]: pgmap v623: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:53.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:53 smithi032 ceph-mon[101299]: pgmap v623: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:53.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:53.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:53 smithi045 ceph-mon[95976]: pgmap v623: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-21T10:12:53.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:53.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:53.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:53.986 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:54.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:54.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:54.121 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:54.158 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:54.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:54.367 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:54.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:54.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:54.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:54.542 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:54.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:54.644 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:54.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.686 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:54.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:54.787 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:54.825 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:54.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:54.927 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:54.962 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:54.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:55.064 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:55.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:55.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:55.209 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:55.255 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:55.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:55.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:55.388 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:55.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:55.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:55.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.526 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:55.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:55.574 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:55 smithi032 ceph-mon[105748]: pgmap v624: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 924 op/s 2024-09-21T10:12:55.574 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2253694269' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:55.574 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:55 smithi032 ceph-mon[101299]: pgmap v624: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 924 op/s 2024-09-21T10:12:55.575 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2253694269' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:55.627 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:55.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:55.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:55.762 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:55.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:55 smithi045 ceph-mon[95976]: pgmap v624: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 924 op/s 2024-09-21T10:12:55.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2253694269' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:55.795 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:55.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:55.898 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:55.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:55.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:56.036 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:56.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.072 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:56.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:56.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:56.195 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:56.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:56.297 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:56.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:56.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:56.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:56.469 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:56.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:56.572 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:56.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:56.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:56.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:56.746 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:56.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:56.848 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:56.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:56.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:56.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:57.020 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:57.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:57.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:12:57.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:12:57.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:12:57.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:12:57.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:12:57.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:12:57.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:12:57.070 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:12:57.070 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:57.070 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:57.070 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:57.070 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:57.070 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:57.070 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:57.070 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:57.070 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:57.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:57.163 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:57.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:57.265 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:57.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.303 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:57.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:57.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:57.444 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:57.545 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:57.576 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:57.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:57.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:57.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:57 smithi032 ceph-mon[105748]: pgmap v625: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-21T10:12:57.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:57 smithi032 ceph-mon[101299]: pgmap v625: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-21T10:12:57.712 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:57.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:57.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:57 smithi045 ceph-mon[95976]: pgmap v625: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-21T10:12:57.814 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:57.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.854 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:57.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:57.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:57.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:57.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:58.090 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:58.134 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:58.235 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:58.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.279 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:58.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:58.380 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:58.413 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:58.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:58.515 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:58.552 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:58.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:58.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:58.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:58.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:58.789 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.824 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:58.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:58.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.965 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:58.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:59.066 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:59.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.103 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:59.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:59.204 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:59.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:59.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.383 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:59.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:59.427 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:12:59.428 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:12:59.428 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:12:59.428 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:12:59.428 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:12:59.484 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:59.506 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:12:59.526 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:59.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:59.628 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:59.662 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:12:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:12:59] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.51.0" 2024-09-21T10:12:59.663 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:59 smithi032 ceph-mon[105748]: pgmap v626: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-21T10:12:59.663 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1947845253' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:59.663 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:12:59 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:59.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:59 smithi032 ceph-mon[101299]: pgmap v626: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-21T10:12:59.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1947845253' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:59.663 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:12:59 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.668 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:12:59.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:59.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:59 smithi045 ceph-mon[95976]: pgmap v626: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-21T10:12:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1947845253' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:12:59 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:59.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.817 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:59.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:59.918 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:59.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.957 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:12:59.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:59.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:13:00.057 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:00.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:00.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:00.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:00.229 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:13:00.229 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:13:00.229 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-09-21T10:13:00.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd in 1 2024-09-21T10:13:00.237 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:00.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:00.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:00.374 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:00.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:00.476 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:00.513 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:00.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:00.565 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:00 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:00.565 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:13:00.565 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:00 smithi032 ceph-mon[101299]: osdmap e214: 8 total, 8 up, 7 in 2024-09-21T10:13:00.565 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:00 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:00.565 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:13:00.565 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:00 smithi032 ceph-mon[105748]: osdmap e214: 8 total, 8 up, 7 in 2024-09-21T10:13:00.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:00.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:00.757 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:00.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:00 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:00.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:13:00.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:00 smithi045 ceph-mon[95976]: osdmap e214: 8 total, 8 up, 7 in 2024-09-21T10:13:00.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:00.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:00.900 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:00.925 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:00.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:13:01.027 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:01.037 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:01.054 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:01.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:01.068 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:01.156 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:01.189 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:01.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:01.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:01.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.327 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:01.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:01.428 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:01.458 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:01.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:01.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:01.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.602 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:01.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:01.678 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:01 smithi032 ceph-mon[105748]: pgmap v628: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-21T10:13:01.679 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:01 smithi032 ceph-mon[101299]: pgmap v628: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-21T10:13:01.703 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:01.738 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:01.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:01 smithi045 ceph-mon[95976]: pgmap v628: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-21T10:13:01.840 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:01.875 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:01.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:01.977 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:02.011 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:02.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:02.113 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:02.142 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:02.244 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:02.277 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:02.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:02.379 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:02.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:02.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:02.469 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:13:02.511 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:02.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.548 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:02.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:02.649 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:02.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.691 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:02.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:02.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:02.835 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:02.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:02.937 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:02.977 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:02.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:03.079 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:03.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.145 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:03.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:03.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:03.282 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:03.384 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:03.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.422 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:03.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:03.523 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:03.555 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:03.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:03.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:03 smithi032 ceph-mon[101299]: pgmap v629: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:13:03.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:03.627 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:03.627 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:03 smithi032 ceph-mon[105748]: pgmap v629: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:13:03.627 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:03.628 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:03.641 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:03.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:03.693 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:03 smithi045 ceph-mon[95976]: pgmap v629: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:13:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:03.794 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:03.826 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:03.928 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:03.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.969 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:03.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:04.070 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:04.111 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:04.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:04.213 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:04.249 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:04.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:04.351 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.387 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:04.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:04.488 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:04.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.525 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:04.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:04.626 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:04.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:04.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:04.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:04.796 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:04.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:04.898 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:04.932 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:04.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:05.034 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:05.078 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:05.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:05.180 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:05.218 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:05.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:05.284 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:13:05.284 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:13:05.316 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:13:05.316 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:13:05.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:05.352 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:13:05.352 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:13:05.355 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:05.393 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:13:05.393 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:13:05.429 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:13:05.430 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:13:05.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:05.464 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:13:05.464 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:13:05.497 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:13:05.497 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:13:05.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:05.541 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:13:05.541 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:13:05.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:05.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.646 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:05.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:05.747 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:05.782 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:05.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:05 smithi045 ceph-mon[95976]: pgmap v630: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:13:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3538934780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:05.884 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:05.915 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:05.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:05.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:05 smithi032 ceph-mon[101299]: pgmap v630: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:13:05.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3538934780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:05.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:05 smithi032 ceph-mon[105748]: pgmap v630: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-21T10:13:05.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3538934780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:06.017 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:06.057 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:06.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:06.159 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:06.197 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:06.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:06.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:06.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:06.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:06.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:06.484 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:06.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:06.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:06.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.627 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:06.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:06.728 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:06.761 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:06.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:06.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:06.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:06.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:07.027 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:07.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:07.129 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:07.166 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:07.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:07.269 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:07.273 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:07.306 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:07.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:07.408 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:07.451 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:07.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:07.553 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:07.597 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:07.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:07.699 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:07.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:07.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:07.841 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:07.876 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:07.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:07 smithi032 ceph-mon[105748]: pgmap v631: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:13:07.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:07 smithi032 ceph-mon[101299]: pgmap v631: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:13:07.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:08.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.019 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:08.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:07 smithi045 ceph-mon[95976]: pgmap v631: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:13:08.120 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:08.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:08.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:08.262 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:08.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.302 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:08.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:08.303 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:08 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:13:08.039+0000 7f532c355640 -1 mon.a@0(leader).osd e214 definitely_dead 0 2024-09-21T10:13:08.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:08.440 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:08.542 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:08.554 INFO:teuthology.orchestra.run.smithi032.stderr:marked in osd.1. 2024-09-21T10:13:08.554 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:08 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/713775438' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:13:08.554 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:08 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/713775438' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:13:08.580 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:08.681 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:08.704 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:08.805 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:08.829 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:08.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:08.930 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:08.952 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:08.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:09.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:13:09.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:08 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/713775438' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:13:09.054 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:09.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:09.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:09.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:09.201 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:09.303 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:09.327 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:09.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:09.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:09.453 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:09.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:09.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:09.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:09.695 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:09.720 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:09.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:09.822 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:09.847 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:09.949 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:13:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:13:09] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.51.0" 2024-09-21T10:13:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:09 smithi032 ceph-mon[105748]: pgmap v632: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:13:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/713775438' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:13:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:09 smithi032 ceph-mon[105748]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T10:13:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:09 smithi032 ceph-mon[101299]: pgmap v632: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:13:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/713775438' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:13:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:09 smithi032 ceph-mon[101299]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T10:13:09.976 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:09.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:09 smithi045 ceph-mon[95976]: pgmap v632: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:13:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/713775438' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:13:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:09 smithi045 ceph-mon[95976]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T10:13:10.078 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:10.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:10.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:10.233 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:10.335 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:10.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:10.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:10.464 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:10.490 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:10.592 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:10.623 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:10.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:10.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:10 smithi032 ceph-mon[101299]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T10:13:10.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:10 smithi032 ceph-mon[101299]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T10:13:10.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:10 smithi032 ceph-mon[105748]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T10:13:10.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:10 smithi032 ceph-mon[105748]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T10:13:10.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:10.750 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:10.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:10.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:10.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:10.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:10.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:11.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:11.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:10 smithi045 ceph-mon[95976]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T10:13:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:10 smithi045 ceph-mon[95976]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T10:13:11.133 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:11.163 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:11.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:11.264 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:11.298 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:11.399 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:11.431 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:11.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:11.442 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:13:11.442 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:13:11.533 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:11.570 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:11.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:11.672 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:11.705 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:11.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[105748]: pgmap v635: 89 pgs: 3 remapped+peering, 3 activating, 10 peering, 1 active+recovering, 72 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 2 B/s, 0 objects/s recovering 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[105748]: 6.b starting backfill to osd.1 from (0'0,0'0] MAX to 214'13142 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[105748]: 6.4 starting backfill to osd.1 from (0'0,0'0] MAX to 214'13047 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[105748]: 6.e starting backfill to osd.1 from (0'0,0'0] MAX to 214'13239 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[105748]: 6.d starting backfill to osd.1 from (0'0,0'0] MAX to 214'12959 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[105748]: 6.1 starting backfill to osd.1 from (0'0,0'0] MAX to 214'13399 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[105748]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[101299]: pgmap v635: 89 pgs: 3 remapped+peering, 3 activating, 10 peering, 1 active+recovering, 72 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 2 B/s, 0 objects/s recovering 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[101299]: 6.b starting backfill to osd.1 from (0'0,0'0] MAX to 214'13142 2024-09-21T10:13:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[101299]: 6.4 starting backfill to osd.1 from (0'0,0'0] MAX to 214'13047 2024-09-21T10:13:11.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[101299]: 6.e starting backfill to osd.1 from (0'0,0'0] MAX to 214'13239 2024-09-21T10:13:11.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[101299]: 6.d starting backfill to osd.1 from (0'0,0'0] MAX to 214'12959 2024-09-21T10:13:11.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[101299]: 6.1 starting backfill to osd.1 from (0'0,0'0] MAX to 214'13399 2024-09-21T10:13:11.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:11 smithi032 ceph-mon[101299]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T10:13:11.806 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:11.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:11.943 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:11.981 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:11.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:11 smithi045 ceph-mon[95976]: pgmap v635: 89 pgs: 3 remapped+peering, 3 activating, 10 peering, 1 active+recovering, 72 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 2 B/s, 0 objects/s recovering 2024-09-21T10:13:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:11 smithi045 ceph-mon[95976]: 6.b starting backfill to osd.1 from (0'0,0'0] MAX to 214'13142 2024-09-21T10:13:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:11 smithi045 ceph-mon[95976]: 6.4 starting backfill to osd.1 from (0'0,0'0] MAX to 214'13047 2024-09-21T10:13:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:11 smithi045 ceph-mon[95976]: 6.e starting backfill to osd.1 from (0'0,0'0] MAX to 214'13239 2024-09-21T10:13:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:11 smithi045 ceph-mon[95976]: 6.d starting backfill to osd.1 from (0'0,0'0] MAX to 214'12959 2024-09-21T10:13:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:11 smithi045 ceph-mon[95976]: 6.1 starting backfill to osd.1 from (0'0,0'0] MAX to 214'13399 2024-09-21T10:13:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:11 smithi045 ceph-mon[95976]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T10:13:12.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:12.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.124 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:12.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:12.184 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-09-21T10:13:12.225 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:12.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.261 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:12.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:12.361 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:12.398 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:12.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:12.500 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:12.535 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:12.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:12.637 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:12.682 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:12.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:12.785 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:12.814 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:12.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:12.890 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:13:12.915 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:12.940 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:12.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:13.041 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:13.045 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:13.068 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:13.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:13.170 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:13.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.199 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:13.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:13.300 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:13.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:13.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:13.441 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:13.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:13.577 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:13.589 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:13:13.620 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:13.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:13.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:13.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.758 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:13.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:13.852 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:13 smithi032 ceph-mon[101299]: pgmap v638: 89 pgs: 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 remapped+peering, 3 activating, 10 peering, 1 active+recovering, 69 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 5918/146958 objects misplaced (4.027%); 5 B/s, 4 objects/s recovering 2024-09-21T10:13:13.852 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:13 smithi032 ceph-mon[101299]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T10:13:13.852 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/954077942' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:13.852 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:13 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:13.852 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:13 smithi032 ceph-mon[105748]: pgmap v638: 89 pgs: 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 remapped+peering, 3 activating, 10 peering, 1 active+recovering, 69 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 5918/146958 objects misplaced (4.027%); 5 B/s, 4 objects/s recovering 2024-09-21T10:13:13.852 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:13 smithi032 ceph-mon[105748]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T10:13:13.852 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/954077942' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:13.852 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:13 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:13.859 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:13.890 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:13.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:13.992 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:14.029 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:14.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:13 smithi045 ceph-mon[95976]: pgmap v638: 89 pgs: 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 remapped+peering, 3 activating, 10 peering, 1 active+recovering, 69 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 5918/146958 objects misplaced (4.027%); 5 B/s, 4 objects/s recovering 2024-09-21T10:13:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:13 smithi045 ceph-mon[95976]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T10:13:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/954077942' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:13 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:14.131 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:14.172 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:14.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:14.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:14.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:14.413 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:14.450 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:14.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:14.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:14.588 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:14.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:14.690 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:14.719 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:14.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:14.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:14.864 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:14.936 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:14 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:14.936 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:14 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:14.936 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:14 smithi045 ceph-mon[95976]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T10:13:14.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:14 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:14.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:14 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:14.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:14 smithi032 ceph-mon[101299]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T10:13:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:14 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:14 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:14 smithi032 ceph-mon[105748]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T10:13:14.966 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:15.000 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:15.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:15.102 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:15.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:15.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:15.164 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:15.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:15.259 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:15.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:15.360 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:15.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.407 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:15.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:15.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:15.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:15.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:15.642 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:15.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:15.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:15 smithi032 ceph-mon[105748]: pgmap v641: 89 pgs: 4 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 remapped+peering, 3 activating, 8 peering, 1 active+recovering, 69 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11754/143814 objects misplaced (8.173%); 0 B/s, 3 objects/s recovering 2024-09-21T10:13:15.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:15 smithi032 ceph-mon[101299]: pgmap v641: 89 pgs: 4 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 remapped+peering, 3 activating, 8 peering, 1 active+recovering, 69 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11754/143814 objects misplaced (8.173%); 0 B/s, 3 objects/s recovering 2024-09-21T10:13:15.775 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:15.809 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:15.911 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:15.943 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:15.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:15 smithi045 ceph-mon[95976]: pgmap v641: 89 pgs: 4 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 remapped+peering, 3 activating, 8 peering, 1 active+recovering, 69 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11754/143814 objects misplaced (8.173%); 0 B/s, 3 objects/s recovering 2024-09-21T10:13:16.045 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:16.078 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:16.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:16.180 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:16.220 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:16.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:16.322 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:16.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:16.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:16.463 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:16.490 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:16.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:16.592 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.636 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:16.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:16.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:13:16.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:16.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:16.880 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:16.911 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:16.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:17.013 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:17.055 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:17.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:17.185 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:13:17.185 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:13:17.185 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:13:17.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:13:17.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.210 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:17.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:17.247 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:13:17.247 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:13:17.285 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:13:17.285 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:13:17.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:17.323 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:13:17.323 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:13:17.338 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:17.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:17.366 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:13:17.366 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:13:17.396 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:13:17.396 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:13:17.432 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:13:17.432 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:13:17.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:17.463 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:13:17.464 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:13:17.502 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:13:17.503 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:13:17.520 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:17.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:17.624 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:17.672 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:17.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:17.774 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:17.819 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:17.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:17.921 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:17.951 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:17 smithi032 ceph-mon[101299]: pgmap v642: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 14106/139056 objects misplaced (10.144%); 6.7 MiB/s, 102 objects/s recovering 2024-09-21T10:13:17.951 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:17 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/600570684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:17.951 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:17 smithi032 ceph-mon[105748]: pgmap v642: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 14106/139056 objects misplaced (10.144%); 6.7 MiB/s, 102 objects/s recovering 2024-09-21T10:13:17.951 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:17 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/600570684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.961 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:17.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:17 smithi045 ceph-mon[95976]: pgmap v642: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 14106/139056 objects misplaced (10.144%); 6.7 MiB/s, 102 objects/s recovering 2024-09-21T10:13:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:17 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/600570684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:18.062 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:18.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:18.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:18.120 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:13:18.120 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:13:18.121 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.121 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:217106 6.2 6:4f2173fa:::benchmark_data_smithi032_171826_object87483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e220)", 2024-09-21T10:13:18.121 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.102585+0000", 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.013836117, 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013867501000000001, 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 217106 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.122 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.102585+0000", 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.102585+0000", 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.102587+0000", 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3710000000000001e-06 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.123 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.102591+0000", 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2100000000000003e-06 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.102603+0000", 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1861e-05 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.102612+0000", 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4839999999999993e-06 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.124 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.104218+0000", 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0016056340000000001 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.104345+0000", 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012702899999999999 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.104461+0000", 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000115699 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.125 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.126 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:13:18.126 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.107917+0000", 2024-09-21T10:13:18.126 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003456953 2024-09-21T10:13:18.126 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.126 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.126 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:18.126 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111794+0000", 2024-09-21T10:13:18.126 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0038762029999999999 2024-09-21T10:13:18.126 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.127 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.127 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.127 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.127 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.127 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44852.0:217105 6.f e220/193 6:fb15ee92:::benchmark_data_smithi032_171826_object87482:head v 220'13639, mlcod=220'13637)", 2024-09-21T10:13:18.127 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.104390+0000", 2024-09-21T10:13:18.127 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.012031541999999999, 2024-09-21T10:13:18.127 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012176938999999999, 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.104390+0000", 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.128 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.129 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.104390+0000", 2024-09-21T10:13:18.129 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.129 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.129 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.129 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.129 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.104391+0000", 2024-09-21T10:13:18.129 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3399999999999998e-07 2024-09-21T10:13:18.129 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.129 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.130 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.130 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.104395+0000", 2024-09-21T10:13:18.130 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6410000000000001e-06 2024-09-21T10:13:18.130 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.130 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.130 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.131 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.104406+0000", 2024-09-21T10:13:18.131 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0417e-05 2024-09-21T10:13:18.131 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.131 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.131 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.131 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.104413+0000", 2024-09-21T10:13:18.132 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8789999999999997e-06 2024-09-21T10:13:18.132 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.132 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.132 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.132 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.106000+0000", 2024-09-21T10:13:18.132 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0015869320000000001 2024-09-21T10:13:18.132 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.133 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.133 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.133 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.106018+0000", 2024-09-21T10:13:18.133 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8783e-05 2024-09-21T10:13:18.133 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.133 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.134 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.134 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.134 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.134 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:217108 6.4 6.e47b0fc4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e220)", 2024-09-21T10:13:18.134 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.105280+0000", 2024-09-21T10:13:18.134 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.011141437000000001, 2024-09-21T10:13:18.135 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011341442, 2024-09-21T10:13:18.135 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.135 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.135 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:13:18.135 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:13:18.135 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:13:18.136 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:13:18.136 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 217108 2024-09-21T10:13:18.136 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.136 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.136 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.136 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.137 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.105280+0000", 2024-09-21T10:13:18.137 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.137 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.137 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.137 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.137 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.105280+0000", 2024-09-21T10:13:18.137 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.138 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.138 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.138 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.138 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.105281+0000", 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.221e-06 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.105285+0000", 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.7409999999999998e-06 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.105297+0000", 2024-09-21T10:13:18.139 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2489e-05 2024-09-21T10:13:18.140 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.140 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.140 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.140 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.105309+0000", 2024-09-21T10:13:18.140 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1751e-05 2024-09-21T10:13:18.140 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.140 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.140 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44852.0:217107 6.f e220/193 6:f379cfd8:::benchmark_data_smithi032_171826_object87484:head v 220'13640, mlcod=220'13637)", 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.106837+0000", 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0095840879999999993, 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0098382959999999998, 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.141 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.106837+0000", 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.106837+0000", 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.142 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.106838+0000", 2024-09-21T10:13:18.143 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8400000000000004e-07 2024-09-21T10:13:18.143 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.143 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.143 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.143 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.106844+0000", 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6500000000000001e-06 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.106853+0000", 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5799999999999998e-06 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.106859+0000", 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9869999999999996e-06 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.144 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.109906+0000", 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0030467649999999999 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.109924+0000", 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8547999999999999e-05 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44852.0:217110 6.5 e220/193 6:a47b86f3:::benchmark_data_smithi032_171826_object87487:head v 220'13459, mlcod=220'13457)", 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.111382+0000", 2024-09-21T10:13:18.145 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0050393640000000002, 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.005347582, 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111382+0000", 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.146 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111382+0000", 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111382+0000", 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4499999999999998e-07 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111386+0000", 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.7239999999999998e-06 2024-09-21T10:13:18.147 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111392+0000", 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0100000000000001e-06 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111396+0000", 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2139999999999998e-06 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111893+0000", 2024-09-21T10:13:18.148 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00049631399999999998 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111910+0000", 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7121000000000001e-05 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44852.0:217098 6.c e220/193 ondisk, result = 0)", 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.111515+0000", 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0049065619999999997, 2024-09-21T10:13:18.149 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.005262989, 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111515+0000", 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.150 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111515+0000", 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111515+0000", 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7499999999999999e-07 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.151 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111517+0000", 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.567e-06 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111522+0000", 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4870000000000002e-06 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.111526+0000", 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5310000000000002e-06 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.152 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116532+0000", 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0050066449999999997 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116539+0000", 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3160000000000002e-06 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44852.0:217114 6.5 e220/193 6:a60e45d3:::benchmark_data_smithi032_171826_object87491:head v 220'13460, mlcod=220'13457)", 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.112583+0000", 2024-09-21T10:13:18.153 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0038383100000000002, 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0042651950000000003, 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.112583+0000", 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.154 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.112583+0000", 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.112584+0000", 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.99e-07 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.112589+0000", 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3589999999999999e-06 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.155 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.112599+0000", 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7310000000000005e-06 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.112605+0000", 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0220000000000001e-06 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.115543+0000", 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0029384939999999998 2024-09-21T10:13:18.156 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.115563+0000", 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9585e-05 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.44852.0:217112 6.f e220/193 6:fc1e5485:::benchmark_data_smithi032_171826_object87489:head v 220'13641, mlcod=220'13637)", 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.113148+0000", 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.003273546, 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0037513490000000002, 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.157 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113148+0000", 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113148+0000", 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.158 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113148+0000", 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.75e-07 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113155+0000", 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0750000000000003e-06 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113165+0000", 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0113e-05 2024-09-21T10:13:18.159 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113171+0000", 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0630000000000002e-06 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113859+0000", 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00068819900000000002 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.160 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113876+0000", 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7258999999999999e-05 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:217115 6.4 6.41f72b34 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e220)", 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.113484+0000", 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0029374119999999999, 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0034676030000000001, 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.161 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 217115 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113484+0000", 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.162 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.163 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113484+0000", 2024-09-21T10:13:18.163 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.163 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.163 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.163 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.163 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113486+0000", 2024-09-21T10:13:18.163 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.649e-06 2024-09-21T10:13:18.163 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.164 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.164 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.164 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113490+0000", 2024-09-21T10:13:18.164 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5959999999999997e-06 2024-09-21T10:13:18.164 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.164 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.165 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.165 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113501+0000", 2024-09-21T10:13:18.165 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1285e-05 2024-09-21T10:13:18.165 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.165 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.165 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.165 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.113508+0000", 2024-09-21T10:13:18.165 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0049999999999998e-06 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44852.0:217103 6.c e220/193)", 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.114695+0000", 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0017264069999999999, 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0023020900000000001, 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.166 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114695+0000", 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114695+0000", 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114696+0000", 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0200000000000002e-07 2024-09-21T10:13:18.167 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114700+0000", 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1250000000000003e-06 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114714+0000", 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4425e-05 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114721+0000", 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8750000000000002e-06 2024-09-21T10:13:18.168 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.44852.0:217103 6.c e220/193)", 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.114757+0000", 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0016645080000000001, 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0022776670000000001, 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.169 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114757+0000", 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114757+0000", 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114757+0000", 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7899999999999999e-07 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.170 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114760+0000", 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.151e-06 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114770+0000", 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1470000000000007e-06 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.114776+0000", 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.003e-06 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.171 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:217119 6.b 6:d97a2808:::benchmark_data_smithi032_171826_object87496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e220)", 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.116728+0000", 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 4294967295.9996934, 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000351243, 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 217119 2024-09-21T10:13:18.172 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116728+0000", 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116728+0000", 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.173 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116730+0000", 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.545e-06 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116735+0000", 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2739999999999998e-06 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116745+0000", 2024-09-21T10:13:18.174 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7240000000000004e-06 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116752+0000", 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2840000000000001e-06 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116837+0000", 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5106999999999997e-05 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116965+0000", 2024-09-21T10:13:18.175 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012768999999999999 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,6,7", 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.117087+0000", 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000121988 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:217100 6.2 6:469c90d8:::benchmark_data_smithi032_171826_object87477:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e220)", 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.096195+0000", 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.020226831000000001, 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.020948187, 2024-09-21T10:13:18.176 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 217100 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.096195+0000", 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.177 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.096195+0000", 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.096196+0000", 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.78e-07 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.096200+0000", 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.758e-06 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.178 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.096215+0000", 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4836e-05 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.096223+0000", 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8949999999999998e-06 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.096598+0000", 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00037544400000000001 2024-09-21T10:13:18.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.096716+0000", 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000117216 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.096827+0000", 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011093400000000001 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.103030+0000", 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.006202974 2024-09-21T10:13:18.180 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.103767+0000", 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00073787500000000003 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:217103 6.c 6:32bc991a:::benchmark_data_smithi032_171826_object87480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e220)", 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:18.097260+0000", 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.019160900000000002, 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.019961274000000001, 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:13:18.181 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 217103 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.097260+0000", 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.182 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.097260+0000", 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.097261+0000", 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9599999999999998e-07 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.097265+0000", 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8010000000000001e-06 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.183 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.097276+0000", 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0964e-05 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.097283+0000", 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1489999999999997e-06 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.108995+0000", 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011711765000000001 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.184 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.109104+0000", 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010956200000000001 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.109207+0000", 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010201599999999999 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.116508+0000", 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0073016469999999997 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:18.185 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:18.117175+0000", 2024-09-21T10:13:18.186 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00066634799999999998 2024-09-21T10:13:18.186 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.186 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:13:18.186 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.186 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:13:18.186 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:13:18.186 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 14 2024-09-21T10:13:18.186 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:13:18.193 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:18.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.225 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:18.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:18.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:18.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:18.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:18.463 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:18.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:18.600 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:18.634 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:18.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:18.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:18.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:18.735 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:18.747 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:18.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.797 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:18.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:18.898 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:18.944 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:19.045 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:19.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.080 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:19.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:19.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:19.230 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:19.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:19.332 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:19.384 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:19.486 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:19.530 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:19.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:19.609 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:13:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:13:19] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-21T10:13:19.632 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:19.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:19.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:19.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:19.814 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:19.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:19.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:19.955 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:19 smithi032 ceph-mon[101299]: pgmap v643: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13537/133908 objects misplaced (10.109%); 5.7 MiB/s, 86 objects/s recovering 2024-09-21T10:13:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:19 smithi032 ceph-mon[105748]: pgmap v643: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13537/133908 objects misplaced (10.109%); 5.7 MiB/s, 86 objects/s recovering 2024-09-21T10:13:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:19 smithi045 ceph-mon[95976]: pgmap v643: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13537/133908 objects misplaced (10.109%); 5.7 MiB/s, 86 objects/s recovering 2024-09-21T10:13:20.056 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:20.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.087 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:20.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:20.107 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:20.189 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:20.235 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:20.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:20.337 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:20.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:20.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:20.480 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:20.526 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:20.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:13:20.628 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.677 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:20.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:20.778 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:20.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.818 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:20.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:20.919 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:20.963 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:20.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:21.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:21.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:21.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:21.209 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:21.242 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:21.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:21.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:13:21.344 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:21.403 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:21.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:21.507 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:21.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:21.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:21.642 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:21.690 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:21 smithi032 ceph-mon[105748]: pgmap v644: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 12055/125550 objects misplaced (9.602%); 10 MiB/s, 159 objects/s recovering 2024-09-21T10:13:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:21 smithi032 ceph-mon[101299]: pgmap v644: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 12055/125550 objects misplaced (9.602%); 10 MiB/s, 159 objects/s recovering 2024-09-21T10:13:21.792 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:21.823 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:21.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:21.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:21.960 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:21.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:21 smithi045 ceph-mon[95976]: pgmap v644: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 12055/125550 objects misplaced (9.602%); 10 MiB/s, 159 objects/s recovering 2024-09-21T10:13:22.062 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:22.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.097 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:22.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:22.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:22.243 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:22.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:22.345 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:22.374 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:22.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:22.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:22.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.532 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:22.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:22.633 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:22.676 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:22.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:22.778 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:22.813 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:22.915 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:22.946 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:22.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:23.049 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:23.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.084 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:23.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:23.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:23.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.230 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:23.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:23.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:23.364 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:23.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:23.466 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:23.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:23.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:23.605 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.648 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:23.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:23.750 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.797 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:23.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:23.898 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.940 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:23.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:23.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:23 smithi032 ceph-mon[105748]: pgmap v645: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 11506/120540 objects misplaced (9.545%); 8.4 MiB/s, 130 objects/s recovering 2024-09-21T10:13:23.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:23 smithi032 ceph-mon[101299]: pgmap v645: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 11506/120540 objects misplaced (9.545%); 8.4 MiB/s, 130 objects/s recovering 2024-09-21T10:13:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:23 smithi045 ceph-mon[95976]: pgmap v645: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 11506/120540 objects misplaced (9.545%); 8.4 MiB/s, 130 objects/s recovering 2024-09-21T10:13:24.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:24.064 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:24.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:24.167 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:24.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:24.313 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:24.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.347 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:24.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:24.448 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:24.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:24.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:24.584 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:24.627 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:24.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:24.634 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:13:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:24 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3423697188' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:13:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:24 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:13:24.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:24 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3423697188' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:13:24.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:24 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:13:24.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:24.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:24.856 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:24.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:24.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:25.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:24 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3423697188' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:13:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:24 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:13:25.061 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:25.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:25.163 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:25.198 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:25.300 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:25.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:25.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:25.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:25.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.507 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:25.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:25.608 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:25.643 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:25.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:25.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:25 smithi032 ceph-mon[105748]: pgmap v646: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 10925/117894 objects misplaced (9.267%); 7.7 MiB/s, 118 objects/s recovering 2024-09-21T10:13:25.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:25 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:25.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3423697188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:13:25.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:25 smithi032 ceph-mon[105748]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T10:13:25.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:25 smithi032 ceph-mon[101299]: pgmap v646: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 10925/117894 objects misplaced (9.267%); 7.7 MiB/s, 118 objects/s recovering 2024-09-21T10:13:25.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:25 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:25.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3423697188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:13:25.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:25 smithi032 ceph-mon[101299]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T10:13:25.734 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:25.735 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:25.745 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:25.801 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:25.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:25.903 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:25.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:25.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:25.991 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:25 smithi045 ceph-mon[95976]: pgmap v646: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 10925/117894 objects misplaced (9.267%); 7.7 MiB/s, 118 objects/s recovering 2024-09-21T10:13:25.991 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:25 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:25.991 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3423697188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:13:25.991 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:25 smithi045 ceph-mon[95976]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T10:13:26.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:26.113 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:26.215 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:26.247 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:26.349 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:26.383 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:26.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:26.484 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:26.514 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:26.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:26.616 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:26.653 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:26.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:26.755 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:26.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:26.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:26.901 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.936 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:26.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:27.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:27.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:27.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:27.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:27.215 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:27.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:27.317 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:27.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:27.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:27.452 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:27.474 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:27.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:27.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.619 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:27.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:27.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:27.758 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:27.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:27.852 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:13:27.856 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:13:27.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:27.879 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:27 smithi032 ceph-mon[101299]: pgmap v648: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 9969/112149 objects misplaced (8.889%); 9.7 MiB/s, 155 objects/s recovering 2024-09-21T10:13:27.879 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:27 smithi032 ceph-mon[105748]: pgmap v648: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 9969/112149 objects misplaced (8.889%); 9.7 MiB/s, 155 objects/s recovering 2024-09-21T10:13:27.888 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:27 smithi045 ceph-mon[95976]: pgmap v648: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 9969/112149 objects misplaced (8.889%); 9.7 MiB/s, 155 objects/s recovering 2024-09-21T10:13:27.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.893 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:27.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:27.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:28.029 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:28.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:28.166 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:28.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:28.245 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:13:28.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:28.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.330 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:28.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:28.431 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:28.460 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:28.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:28.561 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:28.597 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:28.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:28.698 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:28.736 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:28.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:28.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:28 smithi045 ceph-mon[95976]: from='client.45080 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:13:28.838 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:28.869 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:28.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:28.912 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:28 smithi032 ceph-mon[105748]: from='client.45080 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:13:28.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:28 smithi032 ceph-mon[101299]: from='client.45080 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:13:28.941 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":648,"stamp":"2024-09-21T10:13:26.315745+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2452822013,"num_objects":37383,"num_object_clones":0,"num_object_copies":112149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9969,"num_objects_unfound":0,"num_objects_dirty":37383,"num_whiteouts":0,"num_read":6391,"num_read_kb":13846,"num_write":223010,"num_write_kb":8310722,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2237,"num_bytes_recovered":149557172,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119204,"ondisk_log_size":119204,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9825240,"kb_used_data":7329176,"kb_used_omap":35,"kb_used_meta":2495964,"kb_avail":740135976,"statfs":{"total":767960285184,"available":757899239424,"internally_reserved":0,"allocated":7505076224,"data_stored":7496617143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":2555867854},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,9,6,11,7,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-587792384,"num_objects":-8969,"num_object_clones":0,"num_object_copies":-26907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-4137,"num_objects_unfound":0,"num_objects_dirty":-8969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8969,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1552,"num_bytes_recovered":101711872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002977"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":313,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.591134+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:13:13.591134+0000","last_peered":"2024-09-21T10:13:13.591134+0000","last_clean":"2024-09-21T10:13:13.591134+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:13:13.591134+0000","last_undegraded":"2024-09-21T10:13:13.591134+0000","last_fullsized":"2024-09-21T10:13:13.591134+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":188,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.560970+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:13:12.560970+0000","last_peered":"2024-09-21T10:13:12.560970+0000","last_clean":"2024-09-21T10:13:12.560970+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:13:12.560970+0000","last_undegraded":"2024-09-21T10:13:12.560970+0000","last_fullsized":"2024-09-21T10:13:12.560970+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":134,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.562108+0000","last_change":"2024-09-21T10:09:04.982531+0000","last_active":"2024-09-21T10:13:12.562108+0000","last_peered":"2024-09-21T10:13:12.562108+0000","last_clean":"2024-09-21T10:13:12.562108+0000","last_became_active":"2024-09-21T10:09:04.982041+0000","last_became_peered":"2024-09-21T10:09:04.982041+0000","last_unstale":"2024-09-21T10:13:12.562108+0000","last_undegraded":"2024-09-21T10:13:12.562108+0000","last_fullsized":"2024-09-21T10:13:12.562108+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:15.048343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":120,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.560552+0000","last_change":"2024-09-21T10:13:09.583319+0000","last_active":"2024-09-21T10:13:12.560552+0000","last_peered":"2024-09-21T10:13:12.560552+0000","last_clean":"2024-09-21T10:13:12.560552+0000","last_became_active":"2024-09-21T10:13:09.582953+0000","last_became_peered":"2024-09-21T10:13:09.582953+0000","last_unstale":"2024-09-21T10:13:12.560552+0000","last_undegraded":"2024-09-21T10:13:12.560552+0000","last_fullsized":"2024-09-21T10:13:12.560552+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:13.630031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":140,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.590902+0000","last_change":"2024-09-21T10:09:35.958207+0000","last_active":"2024-09-21T10:13:13.590902+0000","last_peered":"2024-09-21T10:13:13.590902+0000","last_clean":"2024-09-21T10:13:13.590902+0000","last_became_active":"2024-09-21T10:09:35.957860+0000","last_became_peered":"2024-09-21T10:09:35.957860+0000","last_unstale":"2024-09-21T10:13:13.590902+0000","last_undegraded":"2024-09-21T10:13:13.590902+0000","last_fullsized":"2024-09-21T10:13:13.590902+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:37.767930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":171,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.565362+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:13:12.565362+0000","last_peered":"2024-09-21T10:13:12.565362+0000","last_clean":"2024-09-21T10:13:12.565362+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:13:12.565362+0000","last_undegraded":"2024-09-21T10:13:12.565362+0000","last_fullsized":"2024-09-21T10:13:12.565362+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":312,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.591362+0000","last_change":"2024-09-21T10:13:09.577549+0000","last_active":"2024-09-21T10:13:13.591362+0000","last_peered":"2024-09-21T10:13:13.591362+0000","last_clean":"2024-09-21T10:13:13.591362+0000","last_became_active":"2024-09-21T10:13:09.577175+0000","last_became_peered":"2024-09-21T10:13:09.577175+0000","last_unstale":"2024-09-21T10:13:13.591362+0000","last_undegraded":"2024-09-21T10:13:13.591362+0000","last_fullsized":"2024-09-21T10:13:13.591362+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:12.412154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":217,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.552357+0000","last_change":"2024-09-21T10:09:04.992336+0000","last_active":"2024-09-21T10:13:12.552357+0000","last_peered":"2024-09-21T10:13:12.552357+0000","last_clean":"2024-09-21T10:13:12.552357+0000","last_became_active":"2024-09-21T10:09:04.981832+0000","last_became_peered":"2024-09-21T10:09:04.981832+0000","last_unstale":"2024-09-21T10:13:12.552357+0000","last_undegraded":"2024-09-21T10:13:12.552357+0000","last_fullsized":"2024-09-21T10:13:12.552357+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:28.114296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"220'13907","reported_seq":13944,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.461317+0000","last_change":"2024-09-21T10:10:08.538702+0000","last_active":"2024-09-21T10:13:24.461317+0000","last_peered":"2024-09-21T10:13:24.461317+0000","last_clean":"2024-09-21T10:13:24.461317+0000","last_became_active":"2024-09-21T10:10:08.538389+0000","last_became_peered":"2024-09-21T10:10:08.538389+0000","last_unstale":"2024-09-21T10:13:24.461317+0000","last_undegraded":"2024-09-21T10:13:24.461317+0000","last_fullsized":"2024-09-21T10:13:24.461317+0000","mapping_epoch":193,"log_start":"208'7000","ondisk_log_start":"208'7000","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6907,"log_dups_size":0,"ondisk_log_size":6907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:48.323929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152895488,"num_objects":2333,"num_object_clones":0,"num_object_copies":6999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13907,"num_write_kb":519680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":227,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.562340+0000","last_change":"2024-09-21T10:13:09.586457+0000","last_active":"2024-09-21T10:13:12.562340+0000","last_peered":"2024-09-21T10:13:12.562340+0000","last_clean":"2024-09-21T10:13:12.562340+0000","last_became_active":"2024-09-21T10:13:09.574624+0000","last_became_peered":"2024-09-21T10:13:09.574624+0000","last_unstale":"2024-09-21T10:13:12.562340+0000","last_undegraded":"2024-09-21T10:13:12.562340+0000","last_fullsized":"2024-09-21T10:13:12.562340+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:39:00.778086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":73,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.586988+0000","last_change":"2024-09-21T10:09:04.976310+0000","last_active":"2024-09-21T10:13:13.586988+0000","last_peered":"2024-09-21T10:13:13.586988+0000","last_clean":"2024-09-21T10:13:13.586988+0000","last_became_active":"2024-09-21T10:09:04.975683+0000","last_became_peered":"2024-09-21T10:09:04.975683+0000","last_unstale":"2024-09-21T10:13:13.586988+0000","last_undegraded":"2024-09-21T10:13:13.586988+0000","last_fullsized":"2024-09-21T10:13:13.586988+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:30:03.905103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"220'13759","reported_seq":13842,"reported_epoch":220,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:13:22.133862+0000","last_change":"2024-09-21T10:13:10.910660+0000","last_active":"2024-09-21T10:13:22.133862+0000","last_peered":"2024-09-21T10:13:22.133862+0000","last_clean":"2024-09-21T10:13:08.554235+0000","last_became_active":"2024-09-21T10:13:10.711708+0000","last_became_peered":"2024-09-21T10:13:10.711708+0000","last_unstale":"2024-09-21T10:13:22.133862+0000","last_undegraded":"2024-09-21T10:13:22.133862+0000","last_fullsized":"2024-09-21T10:13:22.133862+0000","mapping_epoch":216,"log_start":"209'7500","ondisk_log_start":"209'7500","created":193,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6259,"log_dups_size":0,"ondisk_log_size":6259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157483008,"num_objects":2403,"num_object_clones":0,"num_object_copies":7209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2403,"num_objects_unfound":0,"num_objects_dirty":2403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13759,"num_write_kb":517184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,6],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":2403}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":254,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.562956+0000","last_change":"2024-09-21T10:13:09.596445+0000","last_active":"2024-09-21T10:13:12.562956+0000","last_peered":"2024-09-21T10:13:12.562956+0000","last_clean":"2024-09-21T10:13:12.562956+0000","last_became_active":"2024-09-21T10:13:09.582665+0000","last_became_peered":"2024-09-21T10:13:09.582665+0000","last_unstale":"2024-09-21T10:13:12.562956+0000","last_undegraded":"2024-09-21T10:13:12.562956+0000","last_fullsized":"2024-09-21T10:13:12.562956+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:23:47.510351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":219,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.559725+0000","last_change":"2024-09-21T10:13:09.578819+0000","last_active":"2024-09-21T10:13:12.559725+0000","last_peered":"2024-09-21T10:13:12.559725+0000","last_clean":"2024-09-21T10:13:12.559725+0000","last_became_active":"2024-09-21T10:13:09.576707+0000","last_became_peered":"2024-09-21T10:13:09.576707+0000","last_unstale":"2024-09-21T10:13:12.559725+0000","last_undegraded":"2024-09-21T10:13:12.559725+0000","last_fullsized":"2024-09-21T10:13:12.559725+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:19.175846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"220'13797","reported_seq":13834,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.595702+0000","last_change":"2024-09-21T10:10:08.534773+0000","last_active":"2024-09-21T10:13:24.595702+0000","last_peered":"2024-09-21T10:13:24.595702+0000","last_clean":"2024-09-21T10:13:24.595702+0000","last_became_active":"2024-09-21T10:10:08.534442+0000","last_became_peered":"2024-09-21T10:10:08.534442+0000","last_unstale":"2024-09-21T10:13:24.595702+0000","last_undegraded":"2024-09-21T10:13:24.595702+0000","last_fullsized":"2024-09-21T10:13:24.595702+0000","mapping_epoch":193,"log_start":"208'6900","ondisk_log_start":"208'6900","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:25:43.440414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144769024,"num_objects":2209,"num_object_clones":0,"num_object_copies":6627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13797,"num_write_kb":512192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":186,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.562334+0000","last_change":"2024-09-21T10:09:04.986162+0000","last_active":"2024-09-21T10:13:12.562334+0000","last_peered":"2024-09-21T10:13:12.562334+0000","last_clean":"2024-09-21T10:13:12.562334+0000","last_became_active":"2024-09-21T10:09:04.981953+0000","last_became_peered":"2024-09-21T10:09:04.981953+0000","last_unstale":"2024-09-21T10:13:12.562334+0000","last_undegraded":"2024-09-21T10:13:12.562334+0000","last_fullsized":"2024-09-21T10:13:12.562334+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:42.524542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":204,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.552899+0000","last_change":"2024-09-21T10:13:09.578358+0000","last_active":"2024-09-21T10:13:12.552899+0000","last_peered":"2024-09-21T10:13:12.552899+0000","last_clean":"2024-09-21T10:13:12.552899+0000","last_became_active":"2024-09-21T10:13:09.577273+0000","last_became_peered":"2024-09-21T10:13:09.577273+0000","last_unstale":"2024-09-21T10:13:12.552899+0000","last_undegraded":"2024-09-21T10:13:12.552899+0000","last_fullsized":"2024-09-21T10:13:12.552899+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:17.971932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"220'13785","reported_seq":13822,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.505713+0000","last_change":"2024-09-21T10:10:08.535538+0000","last_active":"2024-09-21T10:13:24.505713+0000","last_peered":"2024-09-21T10:13:24.505713+0000","last_clean":"2024-09-21T10:13:24.505713+0000","last_became_active":"2024-09-21T10:10:08.535206+0000","last_became_peered":"2024-09-21T10:10:08.535206+0000","last_unstale":"2024-09-21T10:13:24.505713+0000","last_undegraded":"2024-09-21T10:13:24.505713+0000","last_fullsized":"2024-09-21T10:13:24.505713+0000","mapping_epoch":193,"log_start":"208'6800","ondisk_log_start":"208'6800","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:59.806692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146604032,"num_objects":2237,"num_object_clones":0,"num_object_copies":6711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13785,"num_write_kb":512704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":188,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.597047+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:13:13.597047+0000","last_peered":"2024-09-21T10:13:13.597047+0000","last_clean":"2024-09-21T10:13:13.597047+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:13:13.597047+0000","last_undegraded":"2024-09-21T10:13:13.597047+0000","last_fullsized":"2024-09-21T10:13:13.597047+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":304,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.614884+0000","last_change":"2024-09-21T10:13:09.577231+0000","last_active":"2024-09-21T10:13:13.614884+0000","last_peered":"2024-09-21T10:13:13.614884+0000","last_clean":"2024-09-21T10:13:13.614884+0000","last_became_active":"2024-09-21T10:13:09.576841+0000","last_became_peered":"2024-09-21T10:13:09.576841+0000","last_unstale":"2024-09-21T10:13:13.614884+0000","last_undegraded":"2024-09-21T10:13:13.614884+0000","last_fullsized":"2024-09-21T10:13:13.614884+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:46:23.456779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"220'14010","reported_seq":18271,"reported_epoch":220,"state":"active+remapped+backfilling","last_fresh":"2024-09-21T10:13:24.578025+0000","last_change":"2024-09-21T10:13:10.915099+0000","last_active":"2024-09-21T10:13:24.578025+0000","last_peered":"2024-09-21T10:13:24.578025+0000","last_clean":"2024-09-21T10:13:08.530815+0000","last_became_active":"2024-09-21T10:13:10.704478+0000","last_became_peered":"2024-09-21T10:13:10.704478+0000","last_unstale":"2024-09-21T10:13:24.578025+0000","last_undegraded":"2024-09-21T10:13:24.578025+0000","last_fullsized":"2024-09-21T10:13:24.578025+0000","mapping_epoch":216,"log_start":"205'5700","ondisk_log_start":"205'5700","created":193,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":8310,"log_dups_size":0,"ondisk_log_size":8310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:58.961461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152436736,"num_objects":2326,"num_object_clones":0,"num_object_copies":6978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":502,"num_objects_unfound":0,"num_objects_dirty":2326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14010,"num_write_kb":522752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2101,"num_bytes_recovered":137625600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2,7],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":2326}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":330,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.596790+0000","last_change":"2024-09-21T10:13:09.594302+0000","last_active":"2024-09-21T10:13:13.596790+0000","last_peered":"2024-09-21T10:13:13.596790+0000","last_clean":"2024-09-21T10:13:13.596790+0000","last_became_active":"2024-09-21T10:13:09.576948+0000","last_became_peered":"2024-09-21T10:13:09.576948+0000","last_unstale":"2024-09-21T10:13:13.596790+0000","last_undegraded":"2024-09-21T10:13:13.596790+0000","last_fullsized":"2024-09-21T10:13:13.596790+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:10.360060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":186,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.559581+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:13:12.559581+0000","last_peered":"2024-09-21T10:13:12.559581+0000","last_clean":"2024-09-21T10:13:12.559581+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:13:12.559581+0000","last_undegraded":"2024-09-21T10:13:12.559581+0000","last_fullsized":"2024-09-21T10:13:12.559581+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"220'13968","reported_seq":14034,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:23.639379+0000","last_change":"2024-09-21T10:10:08.532703+0000","last_active":"2024-09-21T10:13:23.639379+0000","last_peered":"2024-09-21T10:13:23.639379+0000","last_clean":"2024-09-21T10:13:23.639379+0000","last_became_active":"2024-09-21T10:10:08.532398+0000","last_became_peered":"2024-09-21T10:10:08.532398+0000","last_unstale":"2024-09-21T10:13:23.639379+0000","last_undegraded":"2024-09-21T10:13:23.639379+0000","last_fullsized":"2024-09-21T10:13:23.639379+0000","mapping_epoch":193,"log_start":"205'5900","ondisk_log_start":"205'5900","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":8068,"log_dups_size":0,"ondisk_log_size":8068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:11.397737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152043520,"num_objects":2320,"num_object_clones":0,"num_object_copies":6960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13968,"num_write_kb":521216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":205,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.553034+0000","last_change":"2024-09-21T10:13:09.578151+0000","last_active":"2024-09-21T10:13:12.553034+0000","last_peered":"2024-09-21T10:13:12.553034+0000","last_clean":"2024-09-21T10:13:12.553034+0000","last_became_active":"2024-09-21T10:13:09.577233+0000","last_became_peered":"2024-09-21T10:13:09.577233+0000","last_unstale":"2024-09-21T10:13:12.553034+0000","last_undegraded":"2024-09-21T10:13:12.553034+0000","last_fullsized":"2024-09-21T10:13:12.553034+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:00.627590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":218,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.604032+0000","last_change":"2024-09-21T10:13:09.583584+0000","last_active":"2024-09-21T10:13:13.604032+0000","last_peered":"2024-09-21T10:13:13.604032+0000","last_clean":"2024-09-21T10:13:13.604032+0000","last_became_active":"2024-09-21T10:13:09.582522+0000","last_became_peered":"2024-09-21T10:13:09.582522+0000","last_unstale":"2024-09-21T10:13:13.604032+0000","last_undegraded":"2024-09-21T10:13:13.604032+0000","last_fullsized":"2024-09-21T10:13:13.604032+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:29:11.080627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.c","version":"220'13804","reported_seq":13870,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:22.113525+0000","last_change":"2024-09-21T10:10:08.532583+0000","last_active":"2024-09-21T10:13:22.113525+0000","last_peered":"2024-09-21T10:13:22.113525+0000","last_clean":"2024-09-21T10:13:22.113525+0000","last_became_active":"2024-09-21T10:10:08.532247+0000","last_became_peered":"2024-09-21T10:10:08.532247+0000","last_unstale":"2024-09-21T10:13:22.113525+0000","last_undegraded":"2024-09-21T10:13:22.113525+0000","last_fullsized":"2024-09-21T10:13:22.113525+0000","mapping_epoch":193,"log_start":"209'7600","ondisk_log_start":"209'7600","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6204,"log_dups_size":0,"ondisk_log_size":6204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:38:14.153767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157024256,"num_objects":2396,"num_object_clones":0,"num_object_copies":7188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13804,"num_write_kb":518400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":187,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.559928+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:13:12.559928+0000","last_peered":"2024-09-21T10:13:12.559928+0000","last_clean":"2024-09-21T10:13:12.559928+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:13:12.559928+0000","last_undegraded":"2024-09-21T10:13:12.559928+0000","last_fullsized":"2024-09-21T10:13:12.559928+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":204,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.561276+0000","last_change":"2024-09-21T10:13:09.579444+0000","last_active":"2024-09-21T10:13:12.561276+0000","last_peered":"2024-09-21T10:13:12.561276+0000","last_clean":"2024-09-21T10:13:12.561276+0000","last_became_active":"2024-09-21T10:13:09.579162+0000","last_became_peered":"2024-09-21T10:13:09.579162+0000","last_unstale":"2024-09-21T10:13:12.561276+0000","last_undegraded":"2024-09-21T10:13:12.561276+0000","last_fullsized":"2024-09-21T10:13:12.561276+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.051244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"220'13736","reported_seq":13800,"reported_epoch":220,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:13:23.616603+0000","last_change":"2024-09-21T10:13:10.775247+0000","last_active":"2024-09-21T10:13:23.616603+0000","last_peered":"2024-09-21T10:13:23.616603+0000","last_clean":"2024-09-21T10:13:08.539356+0000","last_became_active":"2024-09-21T10:13:10.726122+0000","last_became_peered":"2024-09-21T10:13:10.726122+0000","last_unstale":"2024-09-21T10:13:23.616603+0000","last_undegraded":"2024-09-21T10:13:23.616603+0000","last_fullsized":"2024-09-21T10:13:23.616603+0000","mapping_epoch":216,"log_start":"205'5600","ondisk_log_start":"205'5600","created":193,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":8136,"log_dups_size":0,"ondisk_log_size":8136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151191576,"num_objects":2308,"num_object_clones":0,"num_object_copies":6924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2308,"num_objects_unfound":0,"num_objects_dirty":2308,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13736,"num_write_kb":513345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2308}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":314,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.594602+0000","last_change":"2024-09-21T10:13:09.582151+0000","last_active":"2024-09-21T10:13:13.594602+0000","last_peered":"2024-09-21T10:13:13.594602+0000","last_clean":"2024-09-21T10:13:13.594602+0000","last_became_active":"2024-09-21T10:13:09.580224+0000","last_became_peered":"2024-09-21T10:13:09.580224+0000","last_unstale":"2024-09-21T10:13:13.594602+0000","last_undegraded":"2024-09-21T10:13:13.594602+0000","last_fullsized":"2024-09-21T10:13:13.594602+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:15.490461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":22,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.656752+0000","last_change":"2024-09-21T10:13:09.586643+0000","last_active":"2024-09-21T10:13:24.656752+0000","last_peered":"2024-09-21T10:13:24.656752+0000","last_clean":"2024-09-21T10:13:24.656752+0000","last_became_active":"2024-09-21T10:13:09.586283+0000","last_became_peered":"2024-09-21T10:13:09.586283+0000","last_unstale":"2024-09-21T10:13:24.656752+0000","last_undegraded":"2024-09-21T10:13:24.656752+0000","last_fullsized":"2024-09-21T10:13:24.656752+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:20.002373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"220'13875","reported_seq":13913,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.652685+0000","last_change":"2024-09-21T10:10:08.532228+0000","last_active":"2024-09-21T10:13:24.652685+0000","last_peered":"2024-09-21T10:13:24.652685+0000","last_clean":"2024-09-21T10:13:24.652685+0000","last_became_active":"2024-09-21T10:10:08.531882+0000","last_became_peered":"2024-09-21T10:10:08.531882+0000","last_unstale":"2024-09-21T10:13:24.652685+0000","last_undegraded":"2024-09-21T10:13:24.652685+0000","last_fullsized":"2024-09-21T10:13:24.652685+0000","mapping_epoch":193,"log_start":"203'3800","ondisk_log_start":"203'3800","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":10075,"log_dups_size":0,"ondisk_log_size":10075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:55:12.849942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148307968,"num_objects":2263,"num_object_clones":0,"num_object_copies":6789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13875,"num_write_kb":516416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":588,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.574440+0000","last_change":"2024-09-21T10:13:12.573143+0000","last_active":"2024-09-21T10:13:12.574440+0000","last_peered":"2024-09-21T10:13:12.574440+0000","last_clean":"2024-09-21T10:13:12.574440+0000","last_became_active":"2024-09-21T10:13:12.571751+0000","last_became_peered":"2024-09-21T10:13:12.571751+0000","last_unstale":"2024-09-21T10:13:12.574440+0000","last_undegraded":"2024-09-21T10:13:12.574440+0000","last_fullsized":"2024-09-21T10:13:12.574440+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:28.804119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":538,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.596629+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:13:13.596629+0000","last_peered":"2024-09-21T10:13:13.596629+0000","last_clean":"2024-09-21T10:13:13.596629+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:13:13.596629+0000","last_undegraded":"2024-09-21T10:13:13.596629+0000","last_fullsized":"2024-09-21T10:13:13.596629+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":254,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.592768+0000","last_change":"2024-09-21T10:09:23.527489+0000","last_active":"2024-09-21T10:13:13.592768+0000","last_peered":"2024-09-21T10:13:13.592768+0000","last_clean":"2024-09-21T10:13:13.592768+0000","last_became_active":"2024-09-21T10:09:23.526852+0000","last_became_peered":"2024-09-21T10:09:23.526852+0000","last_unstale":"2024-09-21T10:13:13.592768+0000","last_undegraded":"2024-09-21T10:13:13.592768+0000","last_fullsized":"2024-09-21T10:13:13.592768+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:34.378201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":172,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.597995+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:13:13.597995+0000","last_peered":"2024-09-21T10:13:13.597995+0000","last_clean":"2024-09-21T10:13:13.597995+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:13:13.597995+0000","last_undegraded":"2024-09-21T10:13:13.597995+0000","last_fullsized":"2024-09-21T10:13:13.597995+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"220'13715","reported_seq":13761,"reported_epoch":220,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:13:22.122614+0000","last_change":"2024-09-21T10:13:10.836219+0000","last_active":"2024-09-21T10:13:22.122614+0000","last_peered":"2024-09-21T10:13:22.122614+0000","last_clean":"2024-09-21T10:13:08.501556+0000","last_became_active":"2024-09-21T10:13:10.749552+0000","last_became_peered":"2024-09-21T10:13:10.749552+0000","last_unstale":"2024-09-21T10:13:22.122614+0000","last_undegraded":"2024-09-21T10:13:22.122614+0000","last_fullsized":"2024-09-21T10:13:22.122614+0000","mapping_epoch":216,"log_start":"208'6965","ondisk_log_start":"208'6965","created":193,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6750,"log_dups_size":0,"ondisk_log_size":6750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157089792,"num_objects":2397,"num_object_clones":0,"num_object_copies":7191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2397,"num_objects_unfound":0,"num_objects_dirty":2397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13715,"num_write_kb":515584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":2397}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":769,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.587080+0000","last_change":"2024-09-21T10:13:12.573505+0000","last_active":"2024-09-21T10:13:13.587080+0000","last_peered":"2024-09-21T10:13:13.587080+0000","last_clean":"2024-09-21T10:13:13.587080+0000","last_became_active":"2024-09-21T10:13:12.572494+0000","last_became_peered":"2024-09-21T10:13:12.572494+0000","last_unstale":"2024-09-21T10:13:13.587080+0000","last_undegraded":"2024-09-21T10:13:13.587080+0000","last_fullsized":"2024-09-21T10:13:13.587080+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:11.001130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":296,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.614502+0000","last_change":"2024-09-21T10:09:04.987275+0000","last_active":"2024-09-21T10:13:13.614502+0000","last_peered":"2024-09-21T10:13:13.614502+0000","last_clean":"2024-09-21T10:13:13.614502+0000","last_became_active":"2024-09-21T10:09:04.986938+0000","last_became_peered":"2024-09-21T10:09:04.986938+0000","last_unstale":"2024-09-21T10:13:13.614502+0000","last_undegraded":"2024-09-21T10:13:13.614502+0000","last_fullsized":"2024-09-21T10:13:13.614502+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:57:05.058080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":22,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.658639+0000","last_change":"2024-09-21T10:13:09.580568+0000","last_active":"2024-09-21T10:13:24.658639+0000","last_peered":"2024-09-21T10:13:24.658639+0000","last_clean":"2024-09-21T10:13:24.658639+0000","last_became_active":"2024-09-21T10:13:09.579053+0000","last_became_peered":"2024-09-21T10:13:09.579053+0000","last_unstale":"2024-09-21T10:13:24.658639+0000","last_undegraded":"2024-09-21T10:13:24.658639+0000","last_fullsized":"2024-09-21T10:13:24.658639+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:39.206612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"220'13823","reported_seq":13860,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.552346+0000","last_change":"2024-09-21T10:10:08.534712+0000","last_active":"2024-09-21T10:13:24.552346+0000","last_peered":"2024-09-21T10:13:24.552346+0000","last_clean":"2024-09-21T10:13:24.552346+0000","last_became_active":"2024-09-21T10:10:08.534361+0000","last_became_peered":"2024-09-21T10:10:08.534361+0000","last_unstale":"2024-09-21T10:13:24.552346+0000","last_undegraded":"2024-09-21T10:13:24.552346+0000","last_fullsized":"2024-09-21T10:13:24.552346+0000","mapping_epoch":193,"log_start":"208'6900","ondisk_log_start":"208'6900","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6923,"log_dups_size":0,"ondisk_log_size":6923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:46:15.211057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144900096,"num_objects":2211,"num_object_clones":0,"num_object_copies":6633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13823,"num_write_kb":513088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":797,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.600120+0000","last_change":"2024-09-21T10:03:53.840322+0000","last_active":"2024-09-21T10:13:13.600120+0000","last_peered":"2024-09-21T10:13:13.600120+0000","last_clean":"2024-09-21T10:13:13.600120+0000","last_became_active":"2024-09-21T10:03:53.839753+0000","last_became_peered":"2024-09-21T10:03:53.839753+0000","last_unstale":"2024-09-21T10:13:13.600120+0000","last_undegraded":"2024-09-21T10:13:13.600120+0000","last_fullsized":"2024-09-21T10:13:13.600120+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:26.364888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":22,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.652943+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:13:24.652943+0000","last_peered":"2024-09-21T10:13:24.652943+0000","last_clean":"2024-09-21T10:13:24.652943+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:13:24.652943+0000","last_undegraded":"2024-09-21T10:13:24.652943+0000","last_fullsized":"2024-09-21T10:13:24.652943+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":22,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.656849+0000","last_change":"2024-09-21T10:13:09.578776+0000","last_active":"2024-09-21T10:13:24.656849+0000","last_peered":"2024-09-21T10:13:24.656849+0000","last_clean":"2024-09-21T10:13:24.656849+0000","last_became_active":"2024-09-21T10:13:09.578462+0000","last_became_peered":"2024-09-21T10:13:09.578462+0000","last_unstale":"2024-09-21T10:13:24.656849+0000","last_undegraded":"2024-09-21T10:13:24.656849+0000","last_fullsized":"2024-09-21T10:13:24.656849+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:19.035767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"220'13788","reported_seq":13854,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:22.154529+0000","last_change":"2024-09-21T10:10:08.533216+0000","last_active":"2024-09-21T10:13:22.154529+0000","last_peered":"2024-09-21T10:13:22.154529+0000","last_clean":"2024-09-21T10:13:22.154529+0000","last_became_active":"2024-09-21T10:10:08.532943+0000","last_became_peered":"2024-09-21T10:10:08.532943+0000","last_unstale":"2024-09-21T10:13:22.154529+0000","last_undegraded":"2024-09-21T10:13:22.154529+0000","last_fullsized":"2024-09-21T10:13:22.154529+0000","mapping_epoch":193,"log_start":"209'7500","ondisk_log_start":"209'7500","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6288,"log_dups_size":0,"ondisk_log_size":6288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:54.852165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159252480,"num_objects":2430,"num_object_clones":0,"num_object_copies":7290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13788,"num_write_kb":518976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1012,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.596706+0000","last_change":"2024-09-21T10:13:12.560467+0000","last_active":"2024-09-21T10:13:13.596706+0000","last_peered":"2024-09-21T10:13:13.596706+0000","last_clean":"2024-09-21T10:13:13.596706+0000","last_became_active":"2024-09-21T10:13:12.559882+0000","last_became_peered":"2024-09-21T10:13:12.559882+0000","last_unstale":"2024-09-21T10:13:13.596706+0000","last_undegraded":"2024-09-21T10:13:13.596706+0000","last_fullsized":"2024-09-21T10:13:13.596706+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:33.267760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":38,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.599509+0000","last_change":"2024-09-21T10:11:19.927808+0000","last_active":"2024-09-21T10:13:13.599509+0000","last_peered":"2024-09-21T10:13:13.599509+0000","last_clean":"2024-09-21T10:13:13.599509+0000","last_became_active":"2024-09-21T10:11:19.927434+0000","last_became_peered":"2024-09-21T10:11:19.927434+0000","last_unstale":"2024-09-21T10:13:13.599509+0000","last_undegraded":"2024-09-21T10:13:13.599509+0000","last_fullsized":"2024-09-21T10:13:13.599509+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:46:55.178217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":22,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.653773+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:13:24.653773+0000","last_peered":"2024-09-21T10:13:24.653773+0000","last_clean":"2024-09-21T10:13:24.653773+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:13:24.653773+0000","last_undegraded":"2024-09-21T10:13:24.653773+0000","last_fullsized":"2024-09-21T10:13:24.653773+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"220'13883","reported_seq":13949,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:21.400526+0000","last_change":"2024-09-21T10:10:08.529516+0000","last_active":"2024-09-21T10:13:21.400526+0000","last_peered":"2024-09-21T10:13:21.400526+0000","last_clean":"2024-09-21T10:13:21.400526+0000","last_became_active":"2024-09-21T10:10:08.529206+0000","last_became_peered":"2024-09-21T10:10:08.529206+0000","last_unstale":"2024-09-21T10:13:21.400526+0000","last_undegraded":"2024-09-21T10:13:21.400526+0000","last_fullsized":"2024-09-21T10:13:21.400526+0000","mapping_epoch":193,"log_start":"202'3800","ondisk_log_start":"202'3800","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":10083,"log_dups_size":0,"ondisk_log_size":10083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:20.039123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156696576,"num_objects":2391,"num_object_clones":0,"num_object_copies":7173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13883,"num_write_kb":520768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":204,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.645987+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:13:24.645987+0000","last_peered":"2024-09-21T10:13:24.645987+0000","last_clean":"2024-09-21T10:13:24.645987+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:13:24.645987+0000","last_undegraded":"2024-09-21T10:13:24.645987+0000","last_fullsized":"2024-09-21T10:13:24.645987+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":567,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.654079+0000","last_change":"2024-09-21T10:13:12.564835+0000","last_active":"2024-09-21T10:13:24.654079+0000","last_peered":"2024-09-21T10:13:24.654079+0000","last_clean":"2024-09-21T10:13:24.654079+0000","last_became_active":"2024-09-21T10:13:12.564228+0000","last_became_peered":"2024-09-21T10:13:12.564228+0000","last_unstale":"2024-09-21T10:13:24.654079+0000","last_undegraded":"2024-09-21T10:13:24.654079+0000","last_fullsized":"2024-09-21T10:13:24.654079+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:08:56.201381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":22,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.653280+0000","last_change":"2024-09-21T10:13:09.587165+0000","last_active":"2024-09-21T10:13:24.653280+0000","last_peered":"2024-09-21T10:13:24.653280+0000","last_clean":"2024-09-21T10:13:24.653280+0000","last_became_active":"2024-09-21T10:13:09.586186+0000","last_became_peered":"2024-09-21T10:13:09.586186+0000","last_unstale":"2024-09-21T10:13:24.653280+0000","last_undegraded":"2024-09-21T10:13:24.653280+0000","last_fullsized":"2024-09-21T10:13:24.653280+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:10.813133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"220'14109","reported_seq":14160,"reported_epoch":220,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:13:23.655657+0000","last_change":"2024-09-21T10:13:10.903679+0000","last_active":"2024-09-21T10:13:23.655657+0000","last_peered":"2024-09-21T10:13:23.655657+0000","last_clean":"2024-09-21T10:13:08.532560+0000","last_became_active":"2024-09-21T10:13:10.688603+0000","last_became_peered":"2024-09-21T10:13:10.688603+0000","last_unstale":"2024-09-21T10:13:23.655657+0000","last_undegraded":"2024-09-21T10:13:23.655657+0000","last_fullsized":"2024-09-21T10:13:23.655657+0000","mapping_epoch":216,"log_start":"208'7200","ondisk_log_start":"208'7200","created":193,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6909,"log_dups_size":0,"ondisk_log_size":6909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154599424,"num_objects":2359,"num_object_clones":0,"num_object_copies":7077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2359,"num_objects_unfound":0,"num_objects_dirty":2359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14109,"num_write_kb":526976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6,7],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":2359}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":22,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.653981+0000","last_change":"2024-09-21T10:13:09.582222+0000","last_active":"2024-09-21T10:13:24.653981+0000","last_peered":"2024-09-21T10:13:24.653981+0000","last_clean":"2024-09-21T10:13:24.653981+0000","last_became_active":"2024-09-21T10:13:09.581902+0000","last_became_peered":"2024-09-21T10:13:09.581902+0000","last_unstale":"2024-09-21T10:13:24.653981+0000","last_undegraded":"2024-09-21T10:13:24.653981+0000","last_fullsized":"2024-09-21T10:13:24.653981+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:48.342047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":700,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.562745+0000","last_change":"2024-09-21T10:13:09.604050+0000","last_active":"2024-09-21T10:13:12.562745+0000","last_peered":"2024-09-21T10:13:12.562745+0000","last_clean":"2024-09-21T10:13:12.562745+0000","last_became_active":"2024-09-21T10:13:09.583593+0000","last_became_peered":"2024-09-21T10:13:09.583593+0000","last_unstale":"2024-09-21T10:13:12.562745+0000","last_undegraded":"2024-09-21T10:13:12.562745+0000","last_fullsized":"2024-09-21T10:13:12.562745+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:35.420105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":301,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.614801+0000","last_change":"2024-09-21T10:13:09.577273+0000","last_active":"2024-09-21T10:13:13.614801+0000","last_peered":"2024-09-21T10:13:13.614801+0000","last_clean":"2024-09-21T10:13:13.614801+0000","last_became_active":"2024-09-21T10:13:09.576920+0000","last_became_peered":"2024-09-21T10:13:09.576920+0000","last_unstale":"2024-09-21T10:13:13.614801+0000","last_undegraded":"2024-09-21T10:13:13.614801+0000","last_fullsized":"2024-09-21T10:13:13.614801+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:04:39.790942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"220'13790","reported_seq":13856,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:22.107300+0000","last_change":"2024-09-21T10:10:08.532620+0000","last_active":"2024-09-21T10:13:22.107300+0000","last_peered":"2024-09-21T10:13:22.107300+0000","last_clean":"2024-09-21T10:13:22.107300+0000","last_became_active":"2024-09-21T10:10:08.532312+0000","last_became_peered":"2024-09-21T10:10:08.532312+0000","last_unstale":"2024-09-21T10:13:22.107300+0000","last_undegraded":"2024-09-21T10:13:22.107300+0000","last_fullsized":"2024-09-21T10:13:22.107300+0000","mapping_epoch":193,"log_start":"209'7500","ondisk_log_start":"209'7500","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6290,"log_dups_size":0,"ondisk_log_size":6290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:59.315832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158072832,"num_objects":2412,"num_object_clones":0,"num_object_copies":7236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13790,"num_write_kb":518464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":297,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.591563+0000","last_change":"2024-09-21T10:03:53.842610+0000","last_active":"2024-09-21T10:13:13.591563+0000","last_peered":"2024-09-21T10:13:13.591563+0000","last_clean":"2024-09-21T10:13:13.591563+0000","last_became_active":"2024-09-21T10:03:53.841888+0000","last_became_peered":"2024-09-21T10:03:53.841888+0000","last_unstale":"2024-09-21T10:13:13.591563+0000","last_undegraded":"2024-09-21T10:13:13.591563+0000","last_fullsized":"2024-09-21T10:13:13.591563+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:05.833771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":626,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.658854+0000","last_change":"2024-09-21T10:13:12.569309+0000","last_active":"2024-09-21T10:13:24.658854+0000","last_peered":"2024-09-21T10:13:24.658854+0000","last_clean":"2024-09-21T10:13:24.658854+0000","last_became_active":"2024-09-21T10:13:12.568325+0000","last_became_peered":"2024-09-21T10:13:12.568325+0000","last_unstale":"2024-09-21T10:13:24.658854+0000","last_undegraded":"2024-09-21T10:13:24.658854+0000","last_fullsized":"2024-09-21T10:13:24.658854+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:18.314566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":315,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.594382+0000","last_change":"2024-09-21T10:13:09.579331+0000","last_active":"2024-09-21T10:13:13.594382+0000","last_peered":"2024-09-21T10:13:13.594382+0000","last_clean":"2024-09-21T10:13:13.594382+0000","last_became_active":"2024-09-21T10:13:09.578777+0000","last_became_peered":"2024-09-21T10:13:09.578777+0000","last_unstale":"2024-09-21T10:13:13.594382+0000","last_undegraded":"2024-09-21T10:13:13.594382+0000","last_fullsized":"2024-09-21T10:13:13.594382+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:50:34.425710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"220'14138","reported_seq":14175,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.580393+0000","last_change":"2024-09-21T10:10:08.539772+0000","last_active":"2024-09-21T10:13:24.580393+0000","last_peered":"2024-09-21T10:13:24.580393+0000","last_clean":"2024-09-21T10:13:24.580393+0000","last_became_active":"2024-09-21T10:10:08.539432+0000","last_became_peered":"2024-09-21T10:10:08.539432+0000","last_unstale":"2024-09-21T10:13:24.580393+0000","last_undegraded":"2024-09-21T10:13:24.580393+0000","last_fullsized":"2024-09-21T10:13:24.580393+0000","mapping_epoch":193,"log_start":"208'7200","ondisk_log_start":"208'7200","created":193,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:10:07.511128+0000","last_clean_scrub_stamp":"2024-09-21T10:10:07.511128+0000","objects_scrubbed":0,"log_size":6938,"log_dups_size":0,"ondisk_log_size":6938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:17:47.240140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154664960,"num_objects":2360,"num_object_clones":0,"num_object_copies":7080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14138,"num_write_kb":527936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":26,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.657731+0000","last_change":"2024-09-21T10:13:09.592519+0000","last_active":"2024-09-21T10:13:24.657731+0000","last_peered":"2024-09-21T10:13:24.657731+0000","last_clean":"2024-09-21T10:13:24.657731+0000","last_became_active":"2024-09-21T10:13:09.587615+0000","last_became_peered":"2024-09-21T10:13:09.587615+0000","last_unstale":"2024-09-21T10:13:24.657731+0000","last_undegraded":"2024-09-21T10:13:24.657731+0000","last_fullsized":"2024-09-21T10:13:24.657731+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:22:43.216612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":463,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.553019+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:13:12.553019+0000","last_peered":"2024-09-21T10:13:12.553019+0000","last_clean":"2024-09-21T10:13:12.553019+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:13:12.553019+0000","last_undegraded":"2024-09-21T10:13:12.553019+0000","last_fullsized":"2024-09-21T10:13:12.553019+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":172,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.597383+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:13:13.597383+0000","last_peered":"2024-09-21T10:13:13.597383+0000","last_clean":"2024-09-21T10:13:13.597383+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:13:13.597383+0000","last_undegraded":"2024-09-21T10:13:13.597383+0000","last_fullsized":"2024-09-21T10:13:13.597383+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":203,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.559877+0000","last_change":"2024-09-21T10:13:09.578025+0000","last_active":"2024-09-21T10:13:12.559877+0000","last_peered":"2024-09-21T10:13:12.559877+0000","last_clean":"2024-09-21T10:13:12.559877+0000","last_became_active":"2024-09-21T10:13:09.577187+0000","last_became_peered":"2024-09-21T10:13:09.577187+0000","last_unstale":"2024-09-21T10:13:12.559877+0000","last_undegraded":"2024-09-21T10:13:12.559877+0000","last_fullsized":"2024-09-21T10:13:12.559877+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:29:27.483990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":194,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.559740+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:13:12.559740+0000","last_peered":"2024-09-21T10:13:12.559740+0000","last_clean":"2024-09-21T10:13:12.559740+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:13:12.559740+0000","last_undegraded":"2024-09-21T10:13:12.559740+0000","last_fullsized":"2024-09-21T10:13:12.559740+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":22,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.653426+0000","last_change":"2024-09-21T10:13:09.580386+0000","last_active":"2024-09-21T10:13:24.653426+0000","last_peered":"2024-09-21T10:13:24.653426+0000","last_clean":"2024-09-21T10:13:24.653426+0000","last_became_active":"2024-09-21T10:13:09.579170+0000","last_became_peered":"2024-09-21T10:13:09.579170+0000","last_unstale":"2024-09-21T10:13:24.653426+0000","last_undegraded":"2024-09-21T10:13:24.653426+0000","last_fullsized":"2024-09-21T10:13:24.653426+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:51.449950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":189,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.562193+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:13:12.562193+0000","last_peered":"2024-09-21T10:13:12.562193+0000","last_clean":"2024-09-21T10:13:12.562193+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:13:12.562193+0000","last_undegraded":"2024-09-21T10:13:12.562193+0000","last_fullsized":"2024-09-21T10:13:12.562193+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":129,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.562812+0000","last_change":"2024-09-21T10:13:09.584095+0000","last_active":"2024-09-21T10:13:12.562812+0000","last_peered":"2024-09-21T10:13:12.562812+0000","last_clean":"2024-09-21T10:13:12.562812+0000","last_became_active":"2024-09-21T10:13:09.583240+0000","last_became_peered":"2024-09-21T10:13:09.583240+0000","last_unstale":"2024-09-21T10:13:12.562812+0000","last_undegraded":"2024-09-21T10:13:12.562812+0000","last_fullsized":"2024-09-21T10:13:12.562812+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:51.025612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":247,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.590539+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:13:13.590539+0000","last_peered":"2024-09-21T10:13:13.590539+0000","last_clean":"2024-09-21T10:13:13.590539+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:13:13.590539+0000","last_undegraded":"2024-09-21T10:13:13.590539+0000","last_fullsized":"2024-09-21T10:13:13.590539+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":295,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.615151+0000","last_change":"2024-09-21T10:09:35.956536+0000","last_active":"2024-09-21T10:13:13.615151+0000","last_peered":"2024-09-21T10:13:13.615151+0000","last_clean":"2024-09-21T10:13:13.615151+0000","last_became_active":"2024-09-21T10:09:35.956253+0000","last_became_peered":"2024-09-21T10:09:35.956253+0000","last_unstale":"2024-09-21T10:13:13.615151+0000","last_undegraded":"2024-09-21T10:13:13.615151+0000","last_fullsized":"2024-09-21T10:13:13.615151+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:06:06.546355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":326,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.594963+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:13:13.594963+0000","last_peered":"2024-09-21T10:13:13.594963+0000","last_clean":"2024-09-21T10:13:13.594963+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:13:13.594963+0000","last_undegraded":"2024-09-21T10:13:13.594963+0000","last_fullsized":"2024-09-21T10:13:13.594963+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":214,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.594556+0000","last_change":"2024-09-21T10:13:09.573620+0000","last_active":"2024-09-21T10:13:13.594556+0000","last_peered":"2024-09-21T10:13:13.594556+0000","last_clean":"2024-09-21T10:13:13.594556+0000","last_became_active":"2024-09-21T10:13:09.573216+0000","last_became_peered":"2024-09-21T10:13:09.573216+0000","last_unstale":"2024-09-21T10:13:13.594556+0000","last_undegraded":"2024-09-21T10:13:13.594556+0000","last_fullsized":"2024-09-21T10:13:13.594556+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:40.253863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":335,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.594521+0000","last_change":"2024-09-21T10:13:09.581932+0000","last_active":"2024-09-21T10:13:13.594521+0000","last_peered":"2024-09-21T10:13:13.594521+0000","last_clean":"2024-09-21T10:13:13.594521+0000","last_became_active":"2024-09-21T10:13:09.580798+0000","last_became_peered":"2024-09-21T10:13:09.580798+0000","last_unstale":"2024-09-21T10:13:13.594521+0000","last_undegraded":"2024-09-21T10:13:13.594521+0000","last_fullsized":"2024-09-21T10:13:13.594521+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:05.563826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":291,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.615056+0000","last_change":"2024-09-21T10:03:53.844946+0000","last_active":"2024-09-21T10:13:13.615056+0000","last_peered":"2024-09-21T10:13:13.615056+0000","last_clean":"2024-09-21T10:13:13.615056+0000","last_became_active":"2024-09-21T10:03:53.844613+0000","last_became_peered":"2024-09-21T10:03:53.844613+0000","last_unstale":"2024-09-21T10:13:13.615056+0000","last_undegraded":"2024-09-21T10:13:13.615056+0000","last_fullsized":"2024-09-21T10:13:13.615056+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:41.055573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":22,"reported_epoch":221,"state":"active+clean","last_fresh":"2024-09-21T10:13:24.652600+0000","last_change":"2024-09-21T10:13:09.577456+0000","last_active":"2024-09-21T10:13:24.652600+0000","last_peered":"2024-09-21T10:13:24.652600+0000","last_clean":"2024-09-21T10:13:24.652600+0000","last_became_active":"2024-09-21T10:13:09.576161+0000","last_became_peered":"2024-09-21T10:13:09.576161+0000","last_unstale":"2024-09-21T10:13:24.652600+0000","last_undegraded":"2024-09-21T10:13:24.652600+0000","last_fullsized":"2024-09-21T10:13:24.652600+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:43:44.958228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":311,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.593803+0000","last_change":"2024-09-21T10:13:09.580691+0000","last_active":"2024-09-21T10:13:13.593803+0000","last_peered":"2024-09-21T10:13:13.593803+0000","last_clean":"2024-09-21T10:13:13.593803+0000","last_became_active":"2024-09-21T10:13:09.580135+0000","last_became_peered":"2024-09-21T10:13:09.580135+0000","last_unstale":"2024-09-21T10:13:13.593803+0000","last_undegraded":"2024-09-21T10:13:13.593803+0000","last_fullsized":"2024-09-21T10:13:13.593803+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:34:22.880041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":220,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.560194+0000","last_change":"2024-09-21T10:13:09.577994+0000","last_active":"2024-09-21T10:13:12.560194+0000","last_peered":"2024-09-21T10:13:12.560194+0000","last_clean":"2024-09-21T10:13:12.560194+0000","last_became_active":"2024-09-21T10:13:09.577311+0000","last_became_peered":"2024-09-21T10:13:09.577311+0000","last_unstale":"2024-09-21T10:13:12.560194+0000","last_undegraded":"2024-09-21T10:13:12.560194+0000","last_fullsized":"2024-09-21T10:13:12.560194+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:27.532202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":332,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.599222+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:13:13.599222+0000","last_peered":"2024-09-21T10:13:13.599222+0000","last_clean":"2024-09-21T10:13:13.599222+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:13:13.599222+0000","last_undegraded":"2024-09-21T10:13:13.599222+0000","last_fullsized":"2024-09-21T10:13:13.599222+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":99,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.559797+0000","last_change":"2024-09-21T10:13:09.578314+0000","last_active":"2024-09-21T10:13:12.559797+0000","last_peered":"2024-09-21T10:13:12.559797+0000","last_clean":"2024-09-21T10:13:12.559797+0000","last_became_active":"2024-09-21T10:13:09.577570+0000","last_became_peered":"2024-09-21T10:13:09.577570+0000","last_unstale":"2024-09-21T10:13:12.559797+0000","last_undegraded":"2024-09-21T10:13:12.559797+0000","last_fullsized":"2024-09-21T10:13:12.559797+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:21.299713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":65,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.559662+0000","last_change":"2024-09-21T10:13:09.577841+0000","last_active":"2024-09-21T10:13:12.559662+0000","last_peered":"2024-09-21T10:13:12.559662+0000","last_clean":"2024-09-21T10:13:12.559662+0000","last_became_active":"2024-09-21T10:13:09.576793+0000","last_became_peered":"2024-09-21T10:13:09.576793+0000","last_unstale":"2024-09-21T10:13:12.559662+0000","last_undegraded":"2024-09-21T10:13:12.559662+0000","last_fullsized":"2024-09-21T10:13:12.559662+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:12.718520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":215,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.596821+0000","last_change":"2024-09-21T10:13:09.589654+0000","last_active":"2024-09-21T10:13:13.596821+0000","last_peered":"2024-09-21T10:13:13.596821+0000","last_clean":"2024-09-21T10:13:13.596821+0000","last_became_active":"2024-09-21T10:13:09.574438+0000","last_became_peered":"2024-09-21T10:13:09.574438+0000","last_unstale":"2024-09-21T10:13:13.596821+0000","last_undegraded":"2024-09-21T10:13:13.596821+0000","last_fullsized":"2024-09-21T10:13:13.596821+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:12.504161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":209,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.593144+0000","last_change":"2024-09-21T10:09:04.984216+0000","last_active":"2024-09-21T10:13:13.593144+0000","last_peered":"2024-09-21T10:13:13.593144+0000","last_clean":"2024-09-21T10:13:13.593144+0000","last_became_active":"2024-09-21T10:09:04.983830+0000","last_became_peered":"2024-09-21T10:09:04.983830+0000","last_unstale":"2024-09-21T10:13:13.593144+0000","last_undegraded":"2024-09-21T10:13:13.593144+0000","last_fullsized":"2024-09-21T10:13:13.593144+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:42:58.533834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":174,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.596640+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:13:13.596640+0000","last_peered":"2024-09-21T10:13:13.596640+0000","last_clean":"2024-09-21T10:13:13.596640+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:13:13.596640+0000","last_undegraded":"2024-09-21T10:13:13.596640+0000","last_fullsized":"2024-09-21T10:13:13.596640+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":317,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.590981+0000","last_change":"2024-09-21T10:03:53.844043+0000","last_active":"2024-09-21T10:13:13.590981+0000","last_peered":"2024-09-21T10:13:13.590981+0000","last_clean":"2024-09-21T10:13:13.590981+0000","last_became_active":"2024-09-21T10:03:53.842891+0000","last_became_peered":"2024-09-21T10:03:53.842891+0000","last_unstale":"2024-09-21T10:13:13.590981+0000","last_undegraded":"2024-09-21T10:13:13.590981+0000","last_fullsized":"2024-09-21T10:13:13.590981+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:39.891977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":306,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.590994+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:13:13.590994+0000","last_peered":"2024-09-21T10:13:13.590994+0000","last_clean":"2024-09-21T10:13:13.590994+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:13:13.590994+0000","last_undegraded":"2024-09-21T10:13:13.590994+0000","last_fullsized":"2024-09-21T10:13:13.590994+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":203,"reported_epoch":219,"state":"active+clean","last_fresh":"2024-09-21T10:13:12.559636+0000","last_change":"2024-09-21T10:13:09.577388+0000","last_active":"2024-09-21T10:13:12.559636+0000","last_peered":"2024-09-21T10:13:12.559636+0000","last_clean":"2024-09-21T10:13:12.559636+0000","last_became_active":"2024-09-21T10:13:09.577027+0000","last_became_peered":"2024-09-21T10:13:09.577027+0000","last_unstale":"2024-09-21T10:13:12.559636+0000","last_undegraded":"2024-09-21T10:13:12.559636+0000","last_fullsized":"2024-09-21T10:13:12.559636+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:49.522609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":102,"reported_epoch":220,"state":"active+clean","last_fresh":"2024-09-21T10:13:13.599531+0000","last_change":"2024-09-21T10:09:35.961787+0000","last_active":"2024-09-21T10:13:13.599531+0000","last_peered":"2024-09-21T10:13:13.599531+0000","last_clean":"2024-09-21T10:13:13.599531+0000","last_became_active":"2024-09-21T10:09:35.957497+0000","last_became_peered":"2024-09-21T10:09:35.957497+0000","last_unstale":"2024-09-21T10:13:13.599531+0000","last_undegraded":"2024-09-21T10:13:13.599531+0000","last_fullsized":"2024-09-21T10:13:13.599531+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:00.014608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2448031768,"num_objects":37355,"num_object_clones":0,"num_object_copies":112065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9969,"num_objects_unfound":0,"num_objects_dirty":37355,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":221887,"num_write_kb":8295681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2103,"num_bytes_recovered":137625600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7427403776,"data_stored":7427391560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118022,"ondisk_log_size":118022,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":718,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234285,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1682184,"kb_used_data":1326208,"kb_used_omap":1,"kb_used_meta":355966,"kb_avail":92062968,"statfs":{"total":95995035648,"available":94272479232,"internally_reserved":0,"allocated":1358036992,"data_stored":1356969285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":364509643},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495923,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1643020,"kb_used_data":1323140,"kb_used_omap":1,"kb_used_meta":319870,"kb_avail":92102132,"statfs":{"total":95995035648,"available":94312583168,"internally_reserved":0,"allocated":1354895360,"data_stored":1353824367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":327547335},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757560,"num_pgs":12,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":600496,"kb_used_data":294568,"kb_used_omap":1,"kb_used_meta":305918,"kb_avail":93144656,"statfs":{"total":95995035648,"available":95380127744,"internally_reserved":0,"allocated":301637632,"data_stored":300589335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":313260486},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019196,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1058444,"kb_used_data":742660,"kb_used_omap":1,"kb_used_meta":315774,"kb_avail":92686708,"statfs":{"total":95995035648,"available":94911188992,"internally_reserved":0,"allocated":760483840,"data_stored":759412360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":323353030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435380,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2033460,"kb_used_data":1690796,"kb_used_omap":5,"kb_used_meta":342650,"kb_avail":91711692,"statfs":{"total":95995035648,"available":93912772608,"internally_reserved":0,"allocated":1731375104,"data_stored":1730322864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":350873640},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,3,5,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598907,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1895604,"kb_used_data":1512684,"kb_used_omap":7,"kb_used_meta":382904,"kb_avail":91849548,"statfs":{"total":95995035648,"available":94053937152,"internally_reserved":0,"allocated":1548988416,"data_stored":1547936348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":392094157},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762434,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":295604,"kb_used_data":115372,"kb_used_omap":7,"kb_used_meta":180216,"kb_avail":93449548,"statfs":{"total":95995035648,"available":95692337152,"internally_reserved":0,"allocated":118140928,"data_stored":117088840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":184541437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925960,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":616428,"kb_used_data":323748,"kb_used_omap":7,"kb_used_meta":292664,"kb_avail":93128724,"statfs":{"total":95995035648,"available":95363813376,"internally_reserved":0,"allocated":331517952,"data_stored":330473744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":299688126},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":319815680,"data_stored":319815680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":120258560,"data_stored":120258560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1525878784,"data_stored":1525874712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1720385536,"data_stored":1720385536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":749670400,"data_stored":749666328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":298516480,"data_stored":298516480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1349718016,"data_stored":1349713944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1343160320,"data_stored":1343160320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:13:28.946 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 1 2024-09-21T10:13:28.970 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.995 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:28.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:29.095 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:29.121 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:29.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:29.247 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:29.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:29.348 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:29.371 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:29.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:29.473 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:29.494 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:29.596 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:29.620 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:29.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:29.722 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.757 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:29.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:29.848 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:13:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:13:29] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-21T10:13:29.849 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:29 smithi032 ceph-mon[105748]: pgmap v649: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 9477/107280 objects misplaced (8.834%); 9.7 MiB/s, 155 objects/s recovering 2024-09-21T10:13:29.849 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:29 smithi032 ceph-mon[105748]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T10:13:29.849 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:29 smithi032 ceph-mon[105748]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T10:13:29.849 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:29 smithi032 ceph-mon[101299]: pgmap v649: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 9477/107280 objects misplaced (8.834%); 9.7 MiB/s, 155 objects/s recovering 2024-09-21T10:13:29.849 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:29 smithi032 ceph-mon[101299]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T10:13:29.849 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:29 smithi032 ceph-mon[101299]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T10:13:29.858 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:29.907 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:29.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:30.008 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:30.088 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:29 smithi045 ceph-mon[95976]: pgmap v649: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 9477/107280 objects misplaced (8.834%); 9.7 MiB/s, 155 objects/s recovering 2024-09-21T10:13:30.089 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:29 smithi045 ceph-mon[95976]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T10:13:30.089 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:29 smithi045 ceph-mon[95976]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T10:13:30.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:30.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:30.249 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:30.279 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:30.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:30.382 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:30.413 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:30.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:30.545 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:30.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:30.647 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:30.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:30.795 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:30.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.857 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:30.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:30.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:30.997 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:30.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:31.068 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:31.068 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:31.068 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:31.068 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:31.099 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:31.124 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:31.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:31.266 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:31.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:31.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:31.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:31.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:31.482 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:13:31.482 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:13:31.482 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:13:31.482 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:13:31.482 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:13:31.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:31.563 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:31.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:31.655 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:13:31.655 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:13:31.655 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:13:31.655 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:13:31.656 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:13:31.656 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:13:31.656 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:13:31.656 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:13:31.656 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:13:31.656 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:13:31.664 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:31.675 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:13:31.675 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:13:31.712 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:13:31.713 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:13:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:31.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:31.749 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:13:31.749 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:13:31.828 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:31.868 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:31.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:31.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:31 smithi032 ceph-mon[105748]: pgmap v652: 89 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 84 active+clean; 2.0 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 8422/100224 objects misplaced (8.403%); 15 MiB/s, 232 objects/s recovering 2024-09-21T10:13:31.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:31 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/972859812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:31.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:31 smithi032 ceph-mon[101299]: pgmap v652: 89 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 84 active+clean; 2.0 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 8422/100224 objects misplaced (8.403%); 15 MiB/s, 232 objects/s recovering 2024-09-21T10:13:31.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:31 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/972859812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:31.970 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:31.997 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:31.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:32.098 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:32.129 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:32.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:32.231 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:32.273 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:32.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:31 smithi045 ceph-mon[95976]: pgmap v652: 89 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 84 active+clean; 2.0 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 8422/100224 objects misplaced (8.403%); 15 MiB/s, 232 objects/s recovering 2024-09-21T10:13:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:31 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/972859812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:32.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:32.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.420 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:32.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:32.521 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:32.550 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:32.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:32.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:32.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:32.781 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:32.817 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:32.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:32.877 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:13:32.918 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:32.946 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:32.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:33.047 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:33.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.097 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:33.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:33.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:33.245 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:33.347 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:33.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.380 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:33.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:33.481 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:33.507 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:33.609 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:33.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.638 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:33.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:33.739 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:33.775 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:33.876 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.912 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:33.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:34.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:34.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.051 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:34.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:34.152 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:34.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:34.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:33 smithi032 ceph-mon[105748]: pgmap v653: 89 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 84 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 8021/96540 objects misplaced (8.308%); 4.4 MiB/s, 69 objects/s recovering 2024-09-21T10:13:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:34.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:33 smithi032 ceph-mon[101299]: pgmap v653: 89 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 84 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 8021/96540 objects misplaced (8.308%); 4.4 MiB/s, 69 objects/s recovering 2024-09-21T10:13:34.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:33 smithi045 ceph-mon[95976]: pgmap v653: 89 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 84 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 8021/96540 objects misplaced (8.308%); 4.4 MiB/s, 69 objects/s recovering 2024-09-21T10:13:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:34.293 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:34.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:34.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:34.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:34.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.485 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:34.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:34.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:34.620 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:34.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:34.705 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:34.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:34.736 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:34.783 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:34.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:34.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:34.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:34.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.939 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:34.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:35.040 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:35.080 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:35.182 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:35.218 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:35.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:35.319 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:35.352 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:35.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:35.454 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:35.499 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:35.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:35.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:35.640 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:35.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:35.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:35.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:35.882 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:35.912 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:35.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:35 smithi032 ceph-mon[105748]: pgmap v654: 89 pgs: 1 active+remapped+backfilling, 1 peering, 3 active+remapped+backfill_wait, 84 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 6574/94788 objects misplaced (6.935%); 13 MiB/s, 210 objects/s recovering 2024-09-21T10:13:35.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:35 smithi032 ceph-mon[101299]: pgmap v654: 89 pgs: 1 active+remapped+backfilling, 1 peering, 3 active+remapped+backfill_wait, 84 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 6574/94788 objects misplaced (6.935%); 13 MiB/s, 210 objects/s recovering 2024-09-21T10:13:36.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:36.059 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:36.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:36.160 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.200 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:36.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:35 smithi045 ceph-mon[95976]: pgmap v654: 89 pgs: 1 active+remapped+backfilling, 1 peering, 3 active+remapped+backfill_wait, 84 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 6574/94788 objects misplaced (6.935%); 13 MiB/s, 210 objects/s recovering 2024-09-21T10:13:36.301 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.331 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:36.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:36.432 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:36.460 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:36.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:36.562 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:36.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:36.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:13:36.708 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:36.805 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:36.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:36.907 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:36.945 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:37.047 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:37.078 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:37.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:37.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.224 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:37.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:37.324 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:37.358 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:37.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:37.460 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.495 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:37.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:37.596 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:37.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:37.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:37.724 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:37.755 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:37.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:37.858 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:37.892 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:37.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:37.994 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:38.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:38.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:38.061 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:37 smithi032 ceph-mon[105748]: pgmap v655: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6574/89544 objects misplaced (7.342%); 13 MiB/s, 210 objects/s recovering 2024-09-21T10:13:38.061 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:37 smithi032 ceph-mon[101299]: pgmap v655: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6574/89544 objects misplaced (7.342%); 13 MiB/s, 210 objects/s recovering 2024-09-21T10:13:38.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:38.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.195 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:38.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:37 smithi045 ceph-mon[95976]: pgmap v655: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6574/89544 objects misplaced (7.342%); 13 MiB/s, 210 objects/s recovering 2024-09-21T10:13:38.295 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:38.336 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:38.437 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:38.485 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:38.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:38.587 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:38.616 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:38.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:38.717 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:38.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.747 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:38.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:38.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:38.893 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:38.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:38.995 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:39.022 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:39.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:39.124 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:39.155 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:39.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:38 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/534197847' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:13:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:38 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/534197847' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:13:39.256 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:39.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:38 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/534197847' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:13:39.290 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:39.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:39.446 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:39.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:39.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.580 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:39.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:39.681 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:39.710 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:39.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:39.804 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:13:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:13:39] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.51.0" 2024-09-21T10:13:39.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:39.848 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:39.950 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:39.980 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:39.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:40.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:40.115 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:40.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:40.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:39 smithi032 ceph-mon[101299]: pgmap v656: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 6040/84807 objects misplaced (7.122%); 11 MiB/s, 173 objects/s recovering 2024-09-21T10:13:40.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/534197847' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:13:40.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:39 smithi032 ceph-mon[101299]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T10:13:40.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:39 smithi032 ceph-mon[105748]: pgmap v656: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 6040/84807 objects misplaced (7.122%); 11 MiB/s, 173 objects/s recovering 2024-09-21T10:13:40.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/534197847' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:13:40.210 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:39 smithi032 ceph-mon[105748]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T10:13:40.217 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:40.226 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:13:40.226 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:13:40.257 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:40.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:39 smithi045 ceph-mon[95976]: pgmap v656: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 6040/84807 objects misplaced (7.122%); 11 MiB/s, 173 objects/s recovering 2024-09-21T10:13:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:39 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/534197847' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:13:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:39 smithi045 ceph-mon[95976]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T10:13:40.360 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:40.400 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:40.502 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:40.545 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:40.646 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.678 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:40.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:40.761 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 1 pgp_num to 1 2024-09-21T10:13:40.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:40.808 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:40.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:40.910 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:40.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:41.052 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:41.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.075 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:41.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:41.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:41.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:41.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:41.303 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:41.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:41.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:41.451 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:41.552 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:41.618 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:41.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:41.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:41.743 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:41.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:41.877 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:41.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:41.978 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:42.009 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:42.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:42.123 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:41 smithi032 ceph-mon[101299]: pgmap v658: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 5140/76815 objects misplaced (6.691%); 10 MiB/s, 166 objects/s recovering 2024-09-21T10:13:42.123 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:41 smithi032 ceph-mon[101299]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T10:13:42.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:41 smithi032 ceph-mon[105748]: pgmap v658: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 5140/76815 objects misplaced (6.691%); 10 MiB/s, 166 objects/s recovering 2024-09-21T10:13:42.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:41 smithi032 ceph-mon[105748]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T10:13:42.160 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:42.262 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:41 smithi045 ceph-mon[95976]: pgmap v658: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 5140/76815 objects misplaced (6.691%); 10 MiB/s, 166 objects/s recovering 2024-09-21T10:13:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:41 smithi045 ceph-mon[95976]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T10:13:42.311 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:42.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:42.412 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:42.438 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:42.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:42.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:42.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:42.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:42.677 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:42.714 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:42.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:42.815 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:42.854 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:42.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:42.874 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:13:42.956 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:42.984 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:43.086 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:43.114 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:43.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:43.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:42 smithi032 ceph-mon[105748]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T10:13:43.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:42 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/123967346' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:43.124 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:42 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:43.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:42 smithi032 ceph-mon[101299]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T10:13:43.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:42 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/123967346' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:43.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:42 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:43.216 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:43.250 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:43.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:42 smithi045 ceph-mon[95976]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T10:13:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:42 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/123967346' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:42 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:43.353 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:43.389 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:43.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:43.490 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:43.529 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:43.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:43.631 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:43.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.665 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:43.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:43.766 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:43.803 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:43.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:43.904 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:43.941 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:43.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:44.042 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:44.089 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:44.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:44.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:43 smithi045 ceph-mon[95976]: pgmap v661: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 3832/72948 objects misplaced (5.253%); 15 MiB/s, 232 objects/s recovering 2024-09-21T10:13:44.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:43 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:13:44.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:43 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:13:44.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:43 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:44.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:43 smithi045 ceph-mon[95976]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T10:13:44.190 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[105748]: pgmap v661: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 3832/72948 objects misplaced (5.253%); 15 MiB/s, 232 objects/s recovering 2024-09-21T10:13:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:13:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:13:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[105748]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T10:13:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[101299]: pgmap v661: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 3832/72948 objects misplaced (5.253%); 15 MiB/s, 232 objects/s recovering 2024-09-21T10:13:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:13:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:13:44.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:44.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:43 smithi032 ceph-mon[101299]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T10:13:44.224 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:44.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:44.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:44.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:44.463 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:44.502 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:44.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:44.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:44.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:44.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:44.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:44.780 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:44.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:44.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:13:44.881 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.940 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:44.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:45.041 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.094 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:45.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:45.195 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:45.237 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:13:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.253 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:45.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:45.354 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:45.389 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:45.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:45.491 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:45.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.531 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:45.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:45.632 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:45.668 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:45.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:45.747 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:45.770 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:45.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.798 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:45.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:45.899 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:45.931 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:45 smithi032 ceph-mon[101299]: pgmap v663: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 3613/71577 objects misplaced (5.048%); 9.9 MiB/s, 159 objects/s recovering 2024-09-21T10:13:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:45 smithi032 ceph-mon[105748]: pgmap v663: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 3613/71577 objects misplaced (5.048%); 9.9 MiB/s, 159 objects/s recovering 2024-09-21T10:13:46.033 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:45 smithi045 ceph-mon[95976]: pgmap v663: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 3613/71577 objects misplaced (5.048%); 9.9 MiB/s, 159 objects/s recovering 2024-09-21T10:13:46.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:46.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:46.174 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:46.202 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:46.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:46.304 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:46.326 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:46.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:46.427 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:46.465 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:46.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:46.554 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:13:46.554 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:13:46.554 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 2, 3, 4, 5, 6, 7, 1] 2024-09-21T10:13:46.554 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd out 0 2024-09-21T10:13:46.566 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:46.647 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:46.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:46.749 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:46.776 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:46.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:46.877 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:46.917 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:47.019 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:47.058 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:47.160 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:47.202 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:47.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:47.303 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:47.337 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:47.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:47.397 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:13:47.397 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:13:47.439 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:47.460 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:47.562 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:47.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.595 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:47.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:47.696 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:47.718 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:47.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:47.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:13:47.820 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:47.845 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:47.947 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:47.993 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:47.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:47 smithi045 ceph-mon[95976]: pgmap v664: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.3 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3466/66216 objects misplaced (5.234%); 11 MiB/s, 169 objects/s recovering 2024-09-21T10:13:48.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:48.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:13:48.107 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:47 smithi032 ceph-mon[105748]: pgmap v664: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.3 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3466/66216 objects misplaced (5.234%); 11 MiB/s, 169 objects/s recovering 2024-09-21T10:13:48.107 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:47 smithi032 ceph-mon[101299]: pgmap v664: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.3 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3466/66216 objects misplaced (5.234%); 11 MiB/s, 169 objects/s recovering 2024-09-21T10:13:48.121 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:48.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:48.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:48.252 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:48.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:48.350 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:13:48.354 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:48.377 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:48.479 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:48.503 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:48.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:48.605 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:48.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:48.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:48.733 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:48.755 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:48.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:48.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:48.879 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:48.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:48.981 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:49.009 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:49.096 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:48 smithi032 ceph-mon[105748]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T10:13:49.096 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:49.096 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:49.096 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:48 smithi032 ceph-mon[101299]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T10:13:49.096 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:49.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:49.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:49.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:49.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:49.258 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:48 smithi045 ceph-mon[95976]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T10:13:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.297 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:49.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:49.398 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:49.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.445 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:49.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:49.546 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:49.589 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:49.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:49.691 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:49.726 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:49.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:49.828 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:49.869 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:49.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:49.899 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:13:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:13:49] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.51.0" 2024-09-21T10:13:49.922 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:49 smithi045 ceph-mon[95976]: pgmap v666: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2572/61404 objects misplaced (4.189%); 15 MiB/s, 242 objects/s recovering 2024-09-21T10:13:49.923 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:49 smithi045 ceph-mon[95976]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T10:13:49.971 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:50.014 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:50.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:50.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:50.144 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:50.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:50.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:49 smithi032 ceph-mon[105748]: pgmap v666: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2572/61404 objects misplaced (4.189%); 15 MiB/s, 242 objects/s recovering 2024-09-21T10:13:50.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:49 smithi032 ceph-mon[105748]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T10:13:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:49 smithi032 ceph-mon[101299]: pgmap v666: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 86 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2572/61404 objects misplaced (4.189%); 15 MiB/s, 242 objects/s recovering 2024-09-21T10:13:50.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:49 smithi032 ceph-mon[101299]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T10:13:50.245 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:50.280 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:50.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:50.334 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:13:50.335 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:13:50.382 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:50.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.416 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:50.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:50.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:50.549 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:50.561 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:50.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.625 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:50.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:50.726 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:50.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:50.880 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:50.909 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:51.010 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:51.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:13:51.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:51.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:51.149 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.186 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:51.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:51.287 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:51.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:51.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:51.380 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:13:51.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:51.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:51.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:51.581 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:51.606 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:51.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:51.708 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:51.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:51.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:51.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:51.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.924 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:51.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:52.025 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:52.078 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:52.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:52.180 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:51 smithi032 ceph-mon[105748]: pgmap v668: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 2324/54537 objects misplaced (4.261%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T10:13:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:51 smithi032 ceph-mon[101299]: pgmap v668: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 2324/54537 objects misplaced (4.261%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T10:13:52.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:52.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:52.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:52.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:51 smithi045 ceph-mon[95976]: pgmap v668: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 2324/54537 objects misplaced (4.261%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T10:13:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:52.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:52.356 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:52.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:52.458 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:52.491 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:52.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:52.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:52.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.637 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:52.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:52.738 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:52.770 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:52.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:52.872 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:52.901 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:52.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:53.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:52 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:13:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:52 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:13:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:52 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:53.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.041 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:53.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:53.142 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:53.169 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:52 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:13:53.169 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:52 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:13:53.169 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:52 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:53.169 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:52 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:13:53.169 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:52 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:13:53.169 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:52 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:13:53.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:53.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:53.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:53.297 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:53.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:53.314 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:13:53.314 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:13:53.399 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:53.424 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:53.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:53.525 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:53.552 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:53.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:53.653 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:53.678 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:53.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:53.779 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:53.804 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:53.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:53.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.934 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:53.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:53.967 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:13:54.035 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:54.068 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:54.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:54.169 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:53 smithi032 ceph-mon[105748]: pgmap v669: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 2324/53928 objects misplaced (4.309%); 5.5 MiB/s, 87 objects/s recovering 2024-09-21T10:13:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:53 smithi032 ceph-mon[105748]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T10:13:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:54.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:54.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:53 smithi032 ceph-mon[101299]: pgmap v669: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 2324/53928 objects misplaced (4.309%); 5.5 MiB/s, 87 objects/s recovering 2024-09-21T10:13:54.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:53 smithi032 ceph-mon[101299]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T10:13:54.267 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:13:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:53 smithi045 ceph-mon[95976]: pgmap v669: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 2324/53928 objects misplaced (4.309%); 5.5 MiB/s, 87 objects/s recovering 2024-09-21T10:13:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:53 smithi045 ceph-mon[95976]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T10:13:54.310 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:54.351 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:54.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:54.453 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:54.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.478 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:54.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:54.579 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:54.619 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:54.721 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:54.755 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:54.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:54.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:54.883 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:54.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:55.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:54 smithi045 ceph-mon[95976]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T10:13:55.041 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:55.143 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:55.169 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:55.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:54 smithi032 ceph-mon[105748]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T10:13:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:54 smithi032 ceph-mon[101299]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T10:13:55.270 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:55.296 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:55.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:55.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:55.426 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:55.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:55.448 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:13:55.449 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:13:55.489 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:13:55.489 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:13:55.490 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:13:55.358+0000 7f532c355640 -1 mon.a@0(leader).osd e231 definitely_dead 0 2024-09-21T10:13:55.528 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:55.539 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:13:55.539 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:13:55.571 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:13:55.571 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:13:55.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.573 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:55.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:55.609 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:13:55.610 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:13:55.644 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:13:55.644 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:13:55.673 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:55.683 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:13:55.683 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:13:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.719 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:55.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:55.722 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:13:55.722 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:13:55.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:55.852 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:55.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:55.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:55.963 INFO:teuthology.orchestra.run.smithi032.stderr:marked out osd.0. 2024-09-21T10:13:55.967 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:13:55.983 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:55.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:55 smithi045 ceph-mon[95976]: pgmap v672: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 87 active+clean; 1.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 417/50016 objects misplaced (0.834%); 17 MiB/s, 266 objects/s recovering 2024-09-21T10:13:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2543719120' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:13:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:55 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:13:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:55 smithi045 ceph-mon[95976]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-21T10:13:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1807426452' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2618159046' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:13:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:55 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:13:56.085 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:56.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:56.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:56.171 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:13:56.171 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:13:56.172 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 0 2024-09-21T10:13:56.172 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:13:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[101299]: pgmap v672: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 87 active+clean; 1.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 417/50016 objects misplaced (0.834%); 17 MiB/s, 266 objects/s recovering 2024-09-21T10:13:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2543719120' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:13:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:13:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[101299]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-21T10:13:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1807426452' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2618159046' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:13:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:13:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[105748]: pgmap v672: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 87 active+clean; 1.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 417/50016 objects misplaced (0.834%); 17 MiB/s, 266 objects/s recovering 2024-09-21T10:13:56.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2543719120' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:13:56.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:13:56.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[105748]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-21T10:13:56.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1807426452' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:56.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2618159046' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:13:56.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:55 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:13:56.209 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:56.231 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:56.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:56.332 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:56.354 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:56.456 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:56.479 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:56.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:56.580 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:56.602 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:56.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:56.704 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:56.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:56.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:56.761 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:13:56.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:56.850 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:56.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:56.952 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:56.966 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:13:56.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:56.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:56 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:56 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-21T10:13:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:56 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:13:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:56 smithi045 ceph-mon[95976]: osdmap e232: 8 total, 8 up, 7 in 2024-09-21T10:13:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:56 smithi045 ceph-mon[95976]: osdmap e233: 8 total, 8 up, 7 in 2024-09-21T10:13:57.077 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:57.098 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:57.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:57.200 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[101299]: osdmap e232: 8 total, 8 up, 7 in 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[101299]: osdmap e233: 8 total, 8 up, 7 in 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[105748]: osdmap e232: 8 total, 8 up, 7 in 2024-09-21T10:13:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:56 smithi032 ceph-mon[105748]: osdmap e233: 8 total, 8 up, 7 in 2024-09-21T10:13:57.221 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:57.323 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:57.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:57.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:57.448 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:57.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:57.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:57.605 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:57.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:57.707 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:57.730 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:57.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:57.874 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:57.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:57.976 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:58.007 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:58.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:58.108 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:58.133 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:57 smithi032 ceph-mon[105748]: pgmap v674: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 87 active+clean; 920 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 417/44049 objects misplaced (0.947%); 17 MiB/s, 266 objects/s recovering 2024-09-21T10:13:58.133 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:57 smithi032 ceph-mon[105748]: osdmap e234: 8 total, 8 up, 7 in 2024-09-21T10:13:58.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:57 smithi032 ceph-mon[101299]: pgmap v674: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 87 active+clean; 920 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 417/44049 objects misplaced (0.947%); 17 MiB/s, 266 objects/s recovering 2024-09-21T10:13:58.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:57 smithi032 ceph-mon[101299]: osdmap e234: 8 total, 8 up, 7 in 2024-09-21T10:13:58.139 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:58.241 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:58.277 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:58.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:57 smithi045 ceph-mon[95976]: pgmap v674: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 87 active+clean; 920 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 417/44049 objects misplaced (0.947%); 17 MiB/s, 266 objects/s recovering 2024-09-21T10:13:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:57 smithi045 ceph-mon[95976]: osdmap e234: 8 total, 8 up, 7 in 2024-09-21T10:13:58.378 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:58.416 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:58.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:58.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:58.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:58.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:58.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:58.781 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:58.819 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:58.921 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:58.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:13:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:58.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:59.062 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:13:59.063 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:13:59.063 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:13:59.063 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:13:59.063 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:13:59.070 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:59.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.106 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:59.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:59.207 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:59.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:59.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:58 smithi045 ceph-mon[95976]: 6.0 starting backfill to osd.1 from (0'0,0'0] MAX to 231'15455 2024-09-21T10:13:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:58 smithi045 ceph-mon[95976]: 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 231'15414 2024-09-21T10:13:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:58 smithi045 ceph-mon[95976]: osdmap e235: 8 total, 8 up, 7 in 2024-09-21T10:13:59.336 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:59.358 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:58 smithi032 ceph-mon[105748]: 6.0 starting backfill to osd.1 from (0'0,0'0] MAX to 231'15455 2024-09-21T10:13:59.358 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:58 smithi032 ceph-mon[105748]: 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 231'15414 2024-09-21T10:13:59.358 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:58 smithi032 ceph-mon[105748]: osdmap e235: 8 total, 8 up, 7 in 2024-09-21T10:13:59.359 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:58 smithi032 ceph-mon[101299]: 6.0 starting backfill to osd.1 from (0'0,0'0] MAX to 231'15455 2024-09-21T10:13:59.359 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:58 smithi032 ceph-mon[101299]: 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 231'15414 2024-09-21T10:13:59.359 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:58 smithi032 ceph-mon[101299]: osdmap e235: 8 total, 8 up, 7 in 2024-09-21T10:13:59.360 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:59.385 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:59.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:59.487 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:59.514 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:13:59.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:59.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:59.657 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:59.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:59.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:13:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:13:59] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.51.0" 2024-09-21T10:13:59.759 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:59.804 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:59.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:59.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:59.947 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:13:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:00.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:00.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:14:00.091 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:00.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:00.158 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:14:00.192 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:00.217 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:00.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:59 smithi045 ceph-mon[95976]: pgmap v677: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 87 active+clean; 869 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 417/41598 objects misplaced (1.002%); 0 B/s, 0 objects/s recovering 2024-09-21T10:14:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:13:59 smithi045 ceph-mon[95976]: osdmap e236: 8 total, 8 up, 7 in 2024-09-21T10:14:00.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:00.347 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:14:00.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:00.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:00.452 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:00.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:59 smithi032 ceph-mon[105748]: pgmap v677: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 87 active+clean; 869 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 417/41598 objects misplaced (1.002%); 0 B/s, 0 objects/s recovering 2024-09-21T10:14:00.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:13:59 smithi032 ceph-mon[105748]: osdmap e236: 8 total, 8 up, 7 in 2024-09-21T10:14:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:59 smithi032 ceph-mon[101299]: pgmap v677: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 87 active+clean; 869 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 417/41598 objects misplaced (1.002%); 0 B/s, 0 objects/s recovering 2024-09-21T10:14:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:13:59 smithi032 ceph-mon[101299]: osdmap e236: 8 total, 8 up, 7 in 2024-09-21T10:14:00.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.500 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:00.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:00.601 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:00.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:00.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:00.758 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:00.796 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:00.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:00.897 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:00.937 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:00.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:01.039 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:01.083 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:01.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:01.184 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:01.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:01.210 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:01.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:01.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:01.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:01.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:01.461 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:01.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:01.612 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:01.651 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:01.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:01.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:01.792 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:01.894 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:01.952 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:01.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:02.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:02.094 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:02.188 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:01 smithi045 ceph-mon[95976]: pgmap v680: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 86 active+clean; 748 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 490/35751 objects misplaced (1.371%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:14:02.196 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:02.205 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:14:02.205 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:14:02.205 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:14:02.205 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:14:02.205 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233862 5.16 5:6a8e88e5:::benchmark_data_smithi032_140417_object100471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.058459+0000", 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 299.12494566499998, 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0285780770000001, 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233862 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.206 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058459+0000", 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058459+0000", 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058457+0000", 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.207 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058464+0000", 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.109e-06 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058480+0000", 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5707000000000001e-05 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058488+0000", 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7819999999999994e-06 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.208 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058917+0000", 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00042962099999999999 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058929+0000", 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1644999999999999e-05 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965364+0000", 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.90643523400000003 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.209 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965388+0000", 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3691999999999999e-05 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.080586+0000", 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.115198087 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.080700+0000", 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011441400000000001 2024-09-21T10:14:02.210 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.081109+0000", 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000408293 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083462+0000", 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0023531110000000002 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083470+0000", 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9209999999999992e-06 2024-09-21T10:14:02.211 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086650+0000", 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031807570000000002 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086871+0000", 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000221015 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086935+0000", 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3935000000000004e-05 2024-09-21T10:14:02.212 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086969+0000", 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3460000000000002e-05 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087037+0000", 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7974999999999999e-05 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233884 5.16 5:6f45c8b4:::benchmark_data_smithi032_140417_object100493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.213 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.074540+0000", 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 299.10886440500002, 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.015916855, 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233884 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074540+0000", 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.214 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074540+0000", 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074538+0000", 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074545+0000", 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2139999999999999e-06 2024-09-21T10:14:02.215 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074558+0000", 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2448e-05 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074564+0000", 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6590000000000001e-06 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080211+0000", 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0056471289999999999 2024-09-21T10:14:02.216 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080223+0000", 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1331e-05 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965420+0000", 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.88519726399999998 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965430+0000", 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0386e-05 2024-09-21T10:14:02.217 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083497+0000", 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1180668620000001 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083583+0000", 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6243999999999996e-05 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083841+0000", 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00025751600000000001 2024-09-21T10:14:02.218 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086822+0000", 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002981048 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087068+0000", 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024563800000000001 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087100+0000", 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2191999999999999e-05 2024-09-21T10:14:02.219 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090223+0000", 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00312277 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090262+0000", 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8892e-05 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090311+0000", 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9879999999999997e-05 2024-09-21T10:14:02.220 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090457+0000", 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000145418 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233887 5.16 5:6ae9e2ce:::benchmark_data_smithi032_140417_object100496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.076252+0000", 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 299.10715210299998, 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0146629210000002, 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.221 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233887 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076252+0000", 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.222 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076252+0000", 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076251+0000", 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076257+0000", 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4929999999999998e-06 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.223 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076271+0000", 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.451e-05 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076279+0000", 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2259999999999999e-06 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080245+0000", 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039659989999999996 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.224 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080252+0000", 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8800000000000008e-06 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965457+0000", 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.88520482700000003 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965467+0000", 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.859e-06 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.085168+0000", 2024-09-21T10:14:02.225 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.11970049 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.085270+0000", 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000102038 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.085489+0000", 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000219848 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087046+0000", 2024-09-21T10:14:02.226 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001556102 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087128+0000", 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2212999999999994e-05 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087159+0000", 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1071000000000001e-05 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090597+0000", 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0034384329999999999 2024-09-21T10:14:02.227 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090768+0000", 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017031500000000001 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090801+0000", 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3090999999999999e-05 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090915+0000", 2024-09-21T10:14:02.228 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000114504 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233916 5.16 5:68361781:::benchmark_data_smithi032_140417_object100525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.104913+0000", 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 299.07849132899997, 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.986222589, 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.229 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233916 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104913+0000", 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104913+0000", 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.230 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104911+0000", 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999976 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104918+0000", 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8340000000000001e-06 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104932+0000", 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4585e-05 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.231 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104939+0000", 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9709999999999998e-06 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.105003+0000", 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3701000000000006e-05 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.105024+0000", 2024-09-21T10:14:02.232 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0996e-05 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965500+0000", 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.86047598400000003 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965510+0000", 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8509999999999994e-06 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087182+0000", 2024-09-21T10:14:02.233 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.121672279 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087258+0000", 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5883000000000005e-05 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087464+0000", 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000206104 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.234 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090228+0000", 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002764168 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090519+0000", 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000290513 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090568+0000", 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9011000000000002e-05 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.235 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091023+0000", 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00045542999999999999 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091049+0000", 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5609000000000002e-05 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091075+0000", 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.6710999999999999e-05 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091136+0000", 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0202999999999999e-05 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234032 5.16 5:68dad721:::benchmark_data_smithi032_140417_object100641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.92448005599999, 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8323526370000001, 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.237 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234032 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.238 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258928+0000", 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8249999999999998e-06 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258933+0000", 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.239 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258937+0000", 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7699999999999999e-06 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258981+0000", 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3417000000000001e-05 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258988+0000", 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6680000000000001e-06 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.240 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965537+0000", 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.70654863199999995 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965548+0000", 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0576e-05 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.088645+0000", 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1230973209999999 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.241 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.088772+0000", 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012733500000000001 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.088991+0000", 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00021865900000000001 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090530+0000", 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015394339999999999 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.242 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090954+0000", 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00042421599999999999 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090993+0000", 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8735000000000001e-05 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091163+0000", 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017015199999999999 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.243 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091193+0000", 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9189000000000001e-05 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091216+0000", 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3572000000000001e-05 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091277+0000", 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0851999999999999e-05 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.244 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234034 5.16 5:6d7dbd1b:::benchmark_data_smithi032_140417_object100643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.91878112199998, 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.834288012, 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234034 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.245 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.246 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264627+0000", 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4209999999999997e-06 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264638+0000", 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1372000000000001e-05 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264645+0000", 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6409999999999996e-06 2024-09-21T10:14:02.247 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264695+0000", 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9638999999999997e-05 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264705+0000", 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0577999999999999e-05 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965574+0000", 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.70086844199999998 2024-09-21T10:14:02.248 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965583+0000", 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8039999999999996e-06 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091314+0000", 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.125730994 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091445+0000", 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000131056 2024-09-21T10:14:02.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091737+0000", 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00029182999999999998 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098585+0000", 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0068481169999999999 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098634+0000", 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8315000000000001e-05 2024-09-21T10:14:02.250 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098644+0000", 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0589e-05 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098675+0000", 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0474999999999999e-05 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098715+0000", 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0546000000000001e-05 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.251 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098771+0000", 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5798999999999999e-05 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098911+0000", 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000140128 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234086 5.16 5:68c88c0e:::benchmark_data_smithi032_140417_object100695:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.252 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.313647+0000", 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.86975722800003, 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.787847314, 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234086 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313647+0000", 2024-09-21T10:14:02.253 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313647+0000", 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313646+0000", 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313652+0000", 2024-09-21T10:14:02.254 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9190000000000004e-06 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313663+0000", 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1888999999999999e-05 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313669+0000", 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3619999999999997e-06 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313709+0000", 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0435999999999997e-05 2024-09-21T10:14:02.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313719+0000", 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3100000000000006e-06 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965613+0000", 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.65189446500000003 2024-09-21T10:14:02.256 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965623+0000", 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6530000000000006e-06 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.093795+0000", 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1281720960000001 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.257 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.093926+0000", 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013081799999999999 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.094285+0000", 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00035964900000000003 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098651+0000", 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0043658539999999997 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.258 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099003+0000", 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00035174099999999998 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099064+0000", 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1512000000000003e-05 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099119+0000", 2024-09-21T10:14:02.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4803999999999999e-05 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101348+0000", 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022291329999999999 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101409+0000", 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0467000000000002e-05 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101494+0000", 2024-09-21T10:14:02.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5681999999999997e-05 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234124 5.16 5:6c5cf963:::benchmark_data_smithi032_140417_object100733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.351979+0000", 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.831425599, 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.754470526, 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.261 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234124 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351979+0000", 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351979+0000", 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351978+0000", 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351982+0000", 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0729999999999998e-06 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351991+0000", 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4260000000000003e-06 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351996+0000", 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7219999999999996e-06 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.352067+0000", 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0650999999999998e-05 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.352078+0000", 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0991e-05 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.264 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965648+0000", 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.61357046599999998 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965657+0000", 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7949999999999996e-06 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.096078+0000", 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1304205760000001 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.096215+0000", 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000137248 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.096550+0000", 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00033544500000000002 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098924+0000", 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0023735240000000001 2024-09-21T10:14:02.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101244+0000", 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00231985 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101293+0000", 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8739000000000002e-05 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101542+0000", 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024916499999999998 2024-09-21T10:14:02.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106280+0000", 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0047382609999999997 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106326+0000", 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5986999999999999e-05 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106449+0000", 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000123297 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.268 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234166 5.16 5:6ca4e8a5:::benchmark_data_smithi032_140417_object100775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.387858+0000", 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.79554601500001, 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.718943055, 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234166 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.271 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:02.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387858+0000", 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387858+0000", 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387857+0000", 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387863+0000", 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4300000000000003e-06 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387877+0000", 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3878e-05 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387884+0000", 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8839999999999997e-06 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387933+0000", 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8795999999999998e-05 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387941+0000", 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3939999999999996e-06 2024-09-21T10:14:02.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965685+0000", 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57774333 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965693+0000", 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9090000000000006e-06 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099158+0000", 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1334648899999999 2024-09-21T10:14:02.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099306+0000", 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000148076 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099683+0000", 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00037678700000000001 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106241+0000", 2024-09-21T10:14:02.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0065577659999999996 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106512+0000", 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000271301 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106522+0000", 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0155e-05 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106559+0000", 2024-09-21T10:14:02.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6272000000000001e-05 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106608+0000", 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.914e-05 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106668+0000", 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9942000000000003e-05 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106801+0000", 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000133635 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234167 5.16 5:6ce4fd33:::benchmark_data_smithi032_140417_object100776:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.390340+0000", 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.79306455300002, 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.721460698, 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.279 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234167 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390340+0000", 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390340+0000", 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390339+0000", 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390347+0000", 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7729999999999999e-06 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390358+0000", 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1365999999999999e-05 2024-09-21T10:14:02.281 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390363+0000", 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4949999999999999e-06 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390416+0000", 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2979000000000002e-05 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390428+0000", 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1277000000000001e-05 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965719+0000", 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57529121699999997 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968363+0000", 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0026442810000000001 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101578+0000", 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.133214739 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101694+0000", 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115877 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101921+0000", 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000227037 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106824+0000", 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049031949999999999 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106864+0000", 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9711e-05 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106921+0000", 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7426999999999998e-05 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106960+0000", 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8778e-05 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111484+0000", 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0045242879999999996 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111533+0000", 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9286999999999998e-05 2024-09-21T10:14:02.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111801+0000", 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00026706100000000002 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234170 5.16 5:6c81e8e9:::benchmark_data_smithi032_140417_object100779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.396582+0000", 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.78682236499998, 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.720816052, 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.287 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234170 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396582+0000", 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396582+0000", 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396581+0000", 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396585+0000", 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1250000000000003e-06 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396594+0000", 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8049999999999996e-06 2024-09-21T10:14:02.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396598+0000", 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3730000000000003e-06 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396637+0000", 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8633000000000001e-05 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396654+0000", 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6682000000000001e-05 2024-09-21T10:14:02.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968419+0000", 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57176563899999999 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968433+0000", 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3628000000000001e-05 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.103802+0000", 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.135369106 2024-09-21T10:14:02.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.103915+0000", 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011266500000000001 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.104243+0000", 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00032865900000000003 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106878+0000", 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00263496 2024-09-21T10:14:02.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107010+0000", 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013164200000000001 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111444+0000", 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0044338060000000002 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117054+0000", 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0056105570000000004 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117236+0000", 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000182058 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117282+0000", 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6072000000000002e-05 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117398+0000", 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115617 2024-09-21T10:14:02.294 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234173 5.16 5:6d06a077:::benchmark_data_smithi032_140417_object100782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.399962+0000", 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.78344219299998, 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.717757881, 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.295 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234173 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399962+0000", 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399962+0000", 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399961+0000", 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399966+0000", 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2000000000000002e-06 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399977+0000", 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0924000000000001e-05 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399982+0000", 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1399999999999999e-06 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.400024+0000", 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2184999999999997e-05 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.400032+0000", 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1709999999999996e-06 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.298 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968461+0000", 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.56842957699999996 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968471+0000", 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8339999999999994e-06 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107043+0000", 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1385719139999999 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107179+0000", 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013627299999999999 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107562+0000", 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00038266100000000001 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111861+0000", 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0042995639999999996 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.300 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111874+0000", 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2559e-05 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111932+0000", 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7599999999999997e-05 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117167+0000", 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0052358830000000002 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.301 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117589+0000", 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000421063 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117638+0000", 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9394999999999997e-05 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117720+0000", 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2128999999999999e-05 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.302 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234191 5.16 5:6e4c6783:::benchmark_data_smithi032_140417_object100800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.424857+0000", 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.758547104, 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.703213192, 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234191 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.303 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424857+0000", 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424857+0000", 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424856+0000", 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.304 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424862+0000", 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5010000000000001e-06 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424895+0000", 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2545000000000001e-05 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424908+0000", 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3281999999999999e-05 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.305 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424983+0000", 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4461000000000004e-05 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424996+0000", 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3675999999999999e-05 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968498+0000", 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54350163500000004 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968508+0000", 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0185e-05 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.109236+0000", 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.140728183 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.109352+0000", 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115758 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.109580+0000", 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00022780000000000001 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.114551+0000", 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049710209999999999 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117101+0000", 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0025498420000000001 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.308 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117464+0000", 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036382999999999999 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117550+0000", 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5587999999999994e-05 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.127902+0000", 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010351463999999999 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.309 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.127950+0000", 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8401e-05 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128070+0000", 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120559 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234298 5.16 5:6cdfd838:::benchmark_data_smithi032_140417_object100907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.589148+0000", 2024-09-21T10:14:02.310 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.59425624400001, 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5391869969999998, 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234298 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589148+0000", 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.311 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589148+0000", 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589146+0000", 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589153+0000", 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8499999999999996e-06 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589168+0000", 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.447e-05 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589174+0000", 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.003e-06 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589236+0000", 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2528999999999997e-05 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589249+0000", 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2418999999999999e-05 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968535+0000", 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.37928635399999999 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968545+0000", 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6390000000000004e-06 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111967+0000", 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.143422419 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.112085+0000", 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000117673 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.112431+0000", 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00034633500000000002 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117112+0000", 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0046805689999999999 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117503+0000", 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00039122000000000001 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117760+0000", 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00025673999999999998 2024-09-21T10:14:02.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128141+0000", 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010381095999999999 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128180+0000", 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9677e-05 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128223+0000", 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2802000000000001e-05 2024-09-21T10:14:02.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128335+0000", 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011193599999999999 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234305 5.16 5:6f4d2f3d:::benchmark_data_smithi032_140417_object100914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.745057+0000", 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 298.43834704300002, 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3835431599999999, 2024-09-21T10:14:02.318 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234305 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745057+0000", 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745057+0000", 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745056+0000", 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745061+0000", 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6789999999999996e-06 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745071+0000", 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.031e-05 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745076+0000", 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8060000000000002e-06 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745120+0000", 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3581999999999997e-05 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745129+0000", 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3030000000000005e-06 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968592+0000", 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.223463151 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968602+0000", 2024-09-21T10:14:02.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0209e-05 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.114594+0000", 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1459913130000001 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.114739+0000", 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014554999999999999 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:01 smithi032 ceph-mon[105748]: pgmap v680: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 86 active+clean; 748 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 490/35751 objects misplaced (1.371%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:14:02.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:01 smithi032 ceph-mon[101299]: pgmap v680: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 86 active+clean; 748 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 490/35751 objects misplaced (1.371%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.115102+0000", 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036283700000000002 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117514+0000", 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024117330000000001 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117800+0000", 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00028629399999999998 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117833+0000", 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2741999999999999e-05 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128400+0000", 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010566862999999999 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128438+0000", 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8417000000000002e-05 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128488+0000", 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9842999999999997e-05 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128601+0000", 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000112724 2024-09-21T10:14:02.327 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234369 5.e 5:7f9b0c82:::benchmark_data_smithi032_140417_object100978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:05.101285+0000", 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 297.08211933500002, 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0605576990000001, 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.328 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234369 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101285+0000", 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101285+0000", 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.329 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101284+0000", 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101289+0000", 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.011e-06 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101298+0000", 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9730000000000003e-06 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101303+0000", 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1780000000000002e-06 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128798+0000", 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.027495070999999999 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128808+0000", 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5869999999999999e-06 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.149057+0000", 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0202487 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.151882+0000", 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002825634 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2,4", 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.152026+0000", 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014369799999999999 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161477+0000", 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094507810000000001 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161535+0000", 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8399000000000001e-05 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161592+0000", 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6943e-05 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161654+0000", 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2346000000000002e-05 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161711+0000", 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6950999999999997e-05 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161843+0000", 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013147699999999999 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234389 5.e 5:7b3804d8:::benchmark_data_smithi032_140417_object100998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:05.120077+0000", 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 297.06332772600001, 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0524572189999999, 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234389 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.335 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120077+0000", 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120077+0000", 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120076+0000", 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120080+0000", 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2039999999999999e-06 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120088+0000", 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6009999999999995e-06 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120094+0000", 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6540000000000004e-06 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128830+0000", 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087358999999999996 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128838+0000", 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3289999999999994e-06 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.155438+0000", 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0265995969999999 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.155529+0000", 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0626999999999998e-05 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2,4", 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.155640+0000", 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000111137 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161604+0000", 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0059642979999999998 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161886+0000", 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00028169099999999998 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161954+0000", 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8204e-05 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.172342+0000", 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010387828999999999 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.172409+0000", 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6789e-05 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.172534+0000", 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000125339 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234402 5.e 5:7aff253c:::benchmark_data_smithi032_140417_object101011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:05.131926+0000", 2024-09-21T10:14:02.341 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 297.05147840199999, 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0452540100000001, 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234402 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131926+0000", 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.342 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131926+0000", 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131924+0000", 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131931+0000", 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0539999999999997e-06 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131948+0000", 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.683e-05 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131957+0000", 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4989999999999994e-06 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.132233+0000", 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00027579200000000002 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.344 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.132244+0000", 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1243999999999999e-05 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.158689+0000", 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0264447269999999 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.158781+0000", 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2371e-05 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2,4", 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.158857+0000", 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5599000000000006e-05 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161892+0000", 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0030351319999999998 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.162017+0000", 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000125418 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.168766+0000", 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0067484709999999998 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.176992+0000", 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0082261039999999997 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.177039+0000", 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7718e-05 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.347 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.177180+0000", 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014060400000000001 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:238218 5.16 5:6e0183dc:::benchmark_data_smithi032_140417_object104827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:10.001585+0000", 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 292.18181975200002, 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.05259364, 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.348 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 238218 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001585+0000", 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001585+0000", 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.349 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001584+0000", 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001588+0000", 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2710000000000001e-06 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001595+0000", 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5279999999999998e-06 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.350 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001601+0000", 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7459999999999997e-06 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001629+0000", 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7766000000000001e-05 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001702+0000", 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3644000000000002e-05 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:14:02.351 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.002088+0000", 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000385728 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.004328+0000", 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022396650000000001 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.004698+0000", 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036994000000000003 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.352 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.004870+0000", 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017258400000000001 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.016872+0000", 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012001639 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.016887+0000", 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.508e-05 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.353 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:11.054100+0000", 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.037213476 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:11.054178+0000", 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7967999999999996e-05 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:242060 5.e 5:706bc916:::benchmark_data_smithi032_140417_object108669:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:16.562452+0000", 2024-09-21T10:14:02.354 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 285.620952219, 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97550333600000005, 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 242060 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562452+0000", 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.355 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562452+0000", 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562450+0000", 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562457+0000", 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1189999999999999e-06 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:02.356 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562473+0000", 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5197e-05 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562479+0000", 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9739999999999999e-06 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562536+0000", 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7689999999999998e-05 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:02.357 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:16.562547+0000", 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0577000000000001e-05 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.530619+0000", 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96807185799999995 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.530706+0000", 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7257999999999997e-05 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:14:02.358 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.530781+0000", 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4865999999999998e-05 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.533900+0000", 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031187810000000002 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.533906+0000", 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.708e-06 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.537894+0000", 2024-09-21T10:14:02.359 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039874150000000002 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.537933+0000", 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9135999999999999e-05 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:17.537956+0000", 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2603e-05 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:02.360 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:14:02.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:02.400 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:14:02.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:14:02.414 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:02.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:02.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:02.553 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:02.655 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:02.682 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:02.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:02.784 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:02.819 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:02.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:02.921 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:02.965 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:03.005 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:14:03.067 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:03.095 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:03.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:03.197 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:03.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.229 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:03.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:02 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1809470377' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:03.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:14:03.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:03.382 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:03.402 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:02 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1809470377' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:03.402 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:02 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1809470377' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:03.483 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:03.509 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:03.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:03.521 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:14:03.610 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:03.633 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:03.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:03.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:03.760 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:03.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:03.861 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:03.884 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:03.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:03.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:04.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:04.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:04.114 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:04.153 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:04.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:03 smithi045 ceph-mon[95976]: pgmap v681: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 86 active+clean; 748 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 490/35751 objects misplaced (1.371%); 7.9 MiB/s, 121 objects/s recovering 2024-09-21T10:14:04.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:03 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:04.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:03 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1809470377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:04.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:03 smithi045 ceph-mon[95976]: osdmap e237: 8 total, 8 up, 7 in 2024-09-21T10:14:04.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:04.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:04.256 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:04.287 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[105748]: pgmap v681: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 86 active+clean; 748 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 490/35751 objects misplaced (1.371%); 7.9 MiB/s, 121 objects/s recovering 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1809470377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[105748]: osdmap e237: 8 total, 8 up, 7 in 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[101299]: pgmap v681: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 86 active+clean; 748 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 490/35751 objects misplaced (1.371%); 7.9 MiB/s, 121 objects/s recovering 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1809470377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:04.292 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[101299]: osdmap e237: 8 total, 8 up, 7 in 2024-09-21T10:14:04.293 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:04.293 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:04.300 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:04.326 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:14:04.326 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:04.352 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:14:04.352 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:04.373 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:14:04.373 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:04.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:04.396 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:14:04.396 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:04.415 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:04.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:04.418 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:14:04.418 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:04.434 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:14:04.434 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:04.451 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:14:04.451 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:04.468 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:14:04.468 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:04.516 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:04.539 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:04.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:04.640 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:04.662 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:04.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:04.763 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:04.785 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:04.887 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:04.908 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:04.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:05.010 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:05.032 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:05.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:05.134 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:05.164 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:05.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:05.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:05 smithi045 ceph-mon[95976]: osdmap e238: 8 total, 8 up, 7 in 2024-09-21T10:14:05.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/722883202' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:05.298 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:05.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:05.395 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:05 smithi032 ceph-mon[105748]: osdmap e238: 8 total, 8 up, 7 in 2024-09-21T10:14:05.395 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/722883202' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:05.396 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:05 smithi032 ceph-mon[101299]: osdmap e238: 8 total, 8 up, 7 in 2024-09-21T10:14:05.396 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/722883202' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:05.399 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:05.440 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:05.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:05.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:05.588 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:05.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:05.690 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:05.723 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:05.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:05.752 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:14:05.825 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:05.857 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:05.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:05.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:05.980 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:05.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:06.082 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:06.129 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:06.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:06.140 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:06 smithi045 ceph-mon[95976]: pgmap v684: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 658 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 1051/31455 objects misplaced (3.341%); 7.9 MiB/s, 121 objects/s recovering 2024-09-21T10:14:06.140 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:06 smithi045 ceph-mon[95976]: osdmap e239: 8 total, 8 up, 7 in 2024-09-21T10:14:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:06 smithi032 ceph-mon[105748]: pgmap v684: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 658 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 1051/31455 objects misplaced (3.341%); 7.9 MiB/s, 121 objects/s recovering 2024-09-21T10:14:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:06 smithi032 ceph-mon[105748]: osdmap e239: 8 total, 8 up, 7 in 2024-09-21T10:14:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:06 smithi032 ceph-mon[101299]: pgmap v684: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 658 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 1051/31455 objects misplaced (3.341%); 7.9 MiB/s, 121 objects/s recovering 2024-09-21T10:14:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:06 smithi032 ceph-mon[101299]: osdmap e239: 8 total, 8 up, 7 in 2024-09-21T10:14:06.230 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:06.252 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:06.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:06.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:06.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.390 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:06.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:06.437 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:06.438 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:06.491 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:06.527 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:06.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:06.629 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:06.665 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:06.767 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:06.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:06.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:06.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:06.938 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:06.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:07.040 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.078 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:07.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:07.106 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:14:07.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:07.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:07.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:07.330 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:07.369 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:07.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:07.467 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:14:07.471 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:07.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.520 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:07.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:07.621 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:07.654 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:07.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:07.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:07.896 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:07.930 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:07.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:08.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:08.077 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:08.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.094 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:08.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:08.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:08 smithi032 ceph-mon[101299]: pgmap v686: 89 pgs: 1 active+remapped+backfill_wait, 88 active+clean; 532 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 561/25401 objects misplaced (2.209%); 4.6 MiB/s, 72 objects/s recovering 2024-09-21T10:14:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:08 smithi032 ceph-mon[105748]: pgmap v686: 89 pgs: 1 active+remapped+backfill_wait, 88 active+clean; 532 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 561/25401 objects misplaced (2.209%); 4.6 MiB/s, 72 objects/s recovering 2024-09-21T10:14:08.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:08.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:08 smithi045 ceph-mon[95976]: pgmap v686: 89 pgs: 1 active+remapped+backfill_wait, 88 active+clean; 532 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 561/25401 objects misplaced (2.209%); 4.6 MiB/s, 72 objects/s recovering 2024-09-21T10:14:08.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:08.357 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:08.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:08.458 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:08.486 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:08.588 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:08.626 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:08.728 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:08.759 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:08.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:08.861 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:08.895 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:08.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:09.037 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:09.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:09.176 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:09.278 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:09.314 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:09.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:09.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:09.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.446 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:09.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:09.523 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:14:09.546 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:09.610 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:09.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:14:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:14:09] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.51.0" 2024-09-21T10:14:09.712 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:09.741 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:09.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:09.843 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:09.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:09.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:09.990 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:10.015 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:10.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:10.024 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:10.028 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:14:10.066 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:10.066 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:10.117 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:10.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:10 smithi032 ceph-mon[101299]: pgmap v687: 89 pgs: 89 active+clean; 467 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:14:10.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:10 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1369928864' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:14:10.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:10 smithi032 ceph-mon[105748]: pgmap v687: 89 pgs: 89 active+clean; 467 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:14:10.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:10 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1369928864' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:14:10.162 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:10.265 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:10.312 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:10.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:10.414 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.454 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:10.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:10 smithi045 ceph-mon[95976]: pgmap v687: 89 pgs: 89 active+clean; 467 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:14:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:10 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1369928864' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:14:10.554 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:10.576 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:10.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:10.708 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:10.810 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:10.839 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:10.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:10.940 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:10.970 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:10.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:11.072 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:11.100 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:11.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:11.154 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:14:11.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:11.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.245 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:11.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:11.322 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:11 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1369928864' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:14:11.322 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:11 smithi032 ceph-mon[101299]: osdmap e240: 8 total, 8 up, 7 in 2024-09-21T10:14:11.322 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:11 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1369928864' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:14:11.322 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:11 smithi032 ceph-mon[105748]: osdmap e240: 8 total, 8 up, 7 in 2024-09-21T10:14:11.346 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:11.407 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:11 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1369928864' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:14:11.407 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:11 smithi045 ceph-mon[95976]: osdmap e240: 8 total, 8 up, 7 in 2024-09-21T10:14:11.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:11.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:11.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:11.547 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:11.648 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:11.690 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:11.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:11.755 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:14:11.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:11.830 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:11.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:11.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:11.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:11.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:12.059 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:14:12.070 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.102 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:12.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:12.203 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:12.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.246 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:12.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:12.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:12.361 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:12 smithi045 ceph-mon[95976]: pgmap v689: 89 pgs: 89 active+clean; 282 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 949 op/s; 9.4 MiB/s, 149 objects/s recovering 2024-09-21T10:14:12.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:12.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:12 smithi032 ceph-mon[105748]: pgmap v689: 89 pgs: 89 active+clean; 282 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 949 op/s; 9.4 MiB/s, 149 objects/s recovering 2024-09-21T10:14:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:12 smithi032 ceph-mon[101299]: pgmap v689: 89 pgs: 89 active+clean; 282 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 949 op/s; 9.4 MiB/s, 149 objects/s recovering 2024-09-21T10:14:12.482 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:12.524 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:12.626 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.660 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:12.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:12.761 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:12.792 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:12.893 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:12.934 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:12.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:13.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:13.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:13.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:13.087 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:14:13.177 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:13.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:13.315 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:13.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:13.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:13.367 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2419932399' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:14:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2419932399' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:14:13.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2419932399' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:14:13.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:13.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:13.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:13.504 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Removed 129620 objects 2024-09-21T10:14:13.504 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Clean up completed and total clean up time :141.299 2024-09-21T10:14:13.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:13.626 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:13.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:13.728 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:13.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:13.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:13.851 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:13.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:13.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:13.975 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:13.976 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-09-21T10:14:13.976 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-09-21T10:14:14.039 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:14.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:14.062 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:14.142 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:14.166 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:14.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:14 smithi032 ceph-mon[105748]: pgmap v690: 89 pgs: 89 active+clean; 282 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:14:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:14 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:14 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2419932399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:14:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:14 smithi032 ceph-mon[105748]: osdmap e241: 8 total, 8 up, 7 in 2024-09-21T10:14:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:14 smithi032 ceph-mon[101299]: pgmap v690: 89 pgs: 89 active+clean; 282 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:14:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:14 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:14 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2419932399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:14:14.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:14 smithi032 ceph-mon[101299]: osdmap e241: 8 total, 8 up, 7 in 2024-09-21T10:14:14.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:14.274 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:14 smithi045 ceph-mon[95976]: pgmap v690: 89 pgs: 89 active+clean; 282 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:14:14.274 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:14 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:14.274 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:14 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2419932399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:14:14.274 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:14 smithi045 ceph-mon[95976]: osdmap e241: 8 total, 8 up, 7 in 2024-09-21T10:14:14.290 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:14.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:14.301 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:14.301 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:14.391 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:14.414 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:14.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:14.538 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:14.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:14.640 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:14.662 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:14.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:14.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:14.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:14.888 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:14.911 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:14.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:14.971 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:14:15.003 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:14:15.004 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:15.013 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:15.016 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:14:15.016 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:15.036 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:14:15.036 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:15.037 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:15.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:15.052 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:14:15.052 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:15.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:14:15.070 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:15.086 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:14:15.086 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:15.104 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:14:15.104 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:15.122 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:14:15.122 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:15.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:15.162 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:15.179 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:14:15.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:15.272 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:15 smithi032 ceph-mon[101299]: pgmap v692: 89 pgs: 89 active+clean; 170 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 4.0 MiB/s, 63 objects/s recovering 2024-09-21T10:14:15.272 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:15 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2826249354' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:15.272 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:15 smithi032 ceph-mon[105748]: pgmap v692: 89 pgs: 89 active+clean; 170 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 4.0 MiB/s, 63 objects/s recovering 2024-09-21T10:14:15.272 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:15 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2826249354' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:15.280 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:15 smithi045 ceph-mon[95976]: pgmap v692: 89 pgs: 89 active+clean; 170 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 4.0 MiB/s, 63 objects/s recovering 2024-09-21T10:14:15.280 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:15 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2826249354' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:15.286 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:15.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:15.410 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:15.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:15.511 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:15.533 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:15.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:15.634 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:15.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:15.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:15.782 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:15.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:15.884 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:15.905 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:15.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:15.974 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:15.977 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:14:16.007 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:16.041 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:16.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:16.143 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:16.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:16.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:16.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:16.291 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:16.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:16.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:16.418 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:16.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:16.520 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:16.542 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:16.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:16.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:16.665 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:16.767 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:16.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:16.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:16.914 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:16.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:16.984 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:14:16.984 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:14:16.984 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 0 2024-09-21T10:14:16.985 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:14:17.015 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:17.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.038 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:17.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:17.139 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:17.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:17.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:17.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.286 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:17.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:17.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:17.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:17.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:17.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:17.525 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:14:17.535 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:17.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:17.636 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:17.659 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:17.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:17 smithi032 ceph-mon[101299]: pgmap v693: 89 pgs: 89 active+clean; 25 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 884 op/s 2024-09-21T10:14:17.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:17 smithi032 ceph-mon[105748]: pgmap v693: 89 pgs: 89 active+clean; 25 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 884 op/s 2024-09-21T10:14:17.732 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:14:17.761 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:17.783 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:17.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:17 smithi045 ceph-mon[95976]: pgmap v693: 89 pgs: 89 active+clean; 25 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 884 op/s 2024-09-21T10:14:17.885 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:17.907 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:18.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:18.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.032 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:18.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:18.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:18.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:18.258 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:18.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:18.384 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:18.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.407 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:18.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:18.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:18.528 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:14:18.531 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:18.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:18.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:18.655 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:18.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:18.757 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:18.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.785 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:18.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:18.885 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:18.909 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:18.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:19.010 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:19.033 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:19.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:19.134 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:19.158 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:19.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:19.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:19.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:19.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:19.333 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:19.369 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:19.385 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:19.400 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_1' removed 2024-09-21T10:14:19.454 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:19.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:19.506 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:19 smithi045 ceph-mon[95976]: pgmap v694: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 534 op/s 2024-09-21T10:14:19.506 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:19.506 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 5]} : dispatch 2024-09-21T10:14:19.506 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-09-21T10:14:19.506 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [1, 7]} : dispatch 2024-09-21T10:14:19.507 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-09-21T10:14:19.507 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2026773173' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:14:19.556 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:19.578 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:19.635 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:14:19.635 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:14:19.635 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:14:19.635 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:14:19.635 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:14:19.680 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:19.701 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:19.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:19.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:14:19] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.51.0" 2024-09-21T10:14:19.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[105748]: pgmap v694: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 534 op/s 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 5]} : dispatch 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [1, 7]} : dispatch 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2026773173' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[101299]: pgmap v694: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 534 op/s 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 5]} : dispatch 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-09-21T10:14:19.710 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [1, 7]} : dispatch 2024-09-21T10:14:19.711 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-09-21T10:14:19.711 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2026773173' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:14:19.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:19.824 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:19.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:19.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:19.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:20.050 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:20.072 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:20.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:20.173 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:20.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:20.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:20.222 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:14:20.297 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:20.327 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:20.405 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:14:20.418 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:14:20.419 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:14:20.419 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-09-21T10:14:20.419 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_2 16 2024-09-21T10:14:20.428 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:20.451 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:20.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:20.470 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:14:20.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 5]}]': finished 2024-09-21T10:14:20.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-21T10:14:20.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [1, 7]}]': finished 2024-09-21T10:14:20.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-21T10:14:20.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:20 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2026773173' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:14:20.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:20 smithi045 ceph-mon[95976]: osdmap e242: 8 total, 8 up, 7 in 2024-09-21T10:14:20.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:20 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3748587000' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:14:20.553 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:20.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:20.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:20.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:20.698 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:20.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 5]}]': finished 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [1, 7]}]': finished 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2026773173' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[101299]: osdmap e242: 8 total, 8 up, 7 in 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3748587000' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 5]}]': finished 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-21T10:14:20.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [1, 7]}]': finished 2024-09-21T10:14:20.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-21T10:14:20.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2026773173' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:14:20.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[105748]: osdmap e242: 8 total, 8 up, 7 in 2024-09-21T10:14:20.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:20 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3748587000' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:14:20.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:20.822 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:20.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:20.924 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:20.945 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:20.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:21.047 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:21.068 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:21.170 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:21.289 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:21.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:21.391 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:21.417 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:21.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:21.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:21.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:21.646 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:21.669 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:21.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:21 smithi032 ceph-mon[101299]: pgmap v696: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:21 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3748587000' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:14:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:21 smithi032 ceph-mon[101299]: osdmap e243: 8 total, 8 up, 7 in 2024-09-21T10:14:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:21 smithi032 ceph-mon[105748]: pgmap v696: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:21 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3748587000' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:14:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:21 smithi032 ceph-mon[105748]: osdmap e243: 8 total, 8 up, 7 in 2024-09-21T10:14:21.750 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:21 smithi045 ceph-mon[95976]: pgmap v696: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:21.750 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:21 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3748587000' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:14:21.750 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:21 smithi045 ceph-mon[95976]: osdmap e243: 8 total, 8 up, 7 in 2024-09-21T10:14:21.772 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:21.794 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:21.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:21.896 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:21.918 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:22.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.045 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:22.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:22.087 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:14:22.087 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:14:22.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:26440 5.9 5:94853574:::benchmark_data_smithi032_140417_object26439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:05:38.493569+0000", 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 523.58592949299998, 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.164516671, 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.088 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 26440 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493569+0000", 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493569+0000", 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493567+0000", 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493668+0000", 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010119999999999999 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493683+0000", 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4228e-05 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493691+0000", 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8140000000000001e-06 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493802+0000", 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000111934 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.493948+0000", 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000145432 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.494104+0000", 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000156205 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.495291+0000", 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001186566 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.495682+0000", 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00039149900000000001 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.657974+0000", 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16229226399999999 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.658034+0000", 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9431000000000001e-05 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.658086+0000", 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1699000000000001e-05 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:26451 5.9 5:9a35ac15:::benchmark_data_smithi032_140417_object26450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:05:38.496388+0000", 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 523.58311032999995, 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.165390493, 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:22.094 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 26451 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496388+0000", 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496388+0000", 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496387+0000", 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496537+0000", 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014985899999999999 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496548+0000", 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1331e-05 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496554+0000", 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9880000000000001e-06 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496596+0000", 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2224000000000001e-05 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496729+0000", 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013331700000000001 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.496861+0000", 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013177999999999999 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.497956+0000", 2024-09-21T10:14:22.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001095279 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498289+0000", 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00033209500000000002 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661693+0000", 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16340427299999999 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661743+0000", 2024-09-21T10:14:22.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0563999999999999e-05 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661779+0000", 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5138999999999997e-05 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:26453 5.9 5:9800755e:::benchmark_data_smithi032_140417_object26452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:05:38.498323+0000", 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 523.58117525299997, 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16356768499999999, 2024-09-21T10:14:22.100 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 26453 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498323+0000", 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498323+0000", 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498322+0000", 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498394+0000", 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1762999999999997e-05 2024-09-21T10:14:22.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498408+0000", 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3726999999999999e-05 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498413+0000", 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.784e-06 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498441+0000", 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7759e-05 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498528+0000", 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6885000000000006e-05 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.498608+0000", 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9900000000000004e-05 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.499458+0000", 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00084979700000000001 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.499723+0000", 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000265159 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661818+0000", 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16209504399999999 2024-09-21T10:14:22.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661859+0000", 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0960000000000001e-05 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:05:38.661891+0000", 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1968000000000003e-05 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213387 5.10 5.645680d0 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.883648+0000", 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 349.19585005099998, 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99656069300000005, 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213387 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.107 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883648+0000", 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883648+0000", 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883646+0000", 2024-09-21T10:14:22.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:14:22.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883654+0000", 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2479999999999999e-06 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883669+0000", 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5434000000000002e-05 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883677+0000", 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6799999999999993e-06 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883759+0000", 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2169999999999994e-05 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883767+0000", 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1629999999999996e-06 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880198+0000", 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99643097999999997 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880209+0000", 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0875e-05 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213398 5.10 5.bd5fbbf0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.892190+0000", 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 349.18730848299998, 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98813121900000001, 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.112 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213398 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892190+0000", 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892190+0000", 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892189+0000", 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892194+0000", 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9169999999999997e-06 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892205+0000", 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1073000000000001e-05 2024-09-21T10:14:22.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892213+0000", 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9379999999999992e-06 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892273+0000", 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0674999999999998e-05 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892284+0000", 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0335e-05 2024-09-21T10:14:22.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880308+0000", 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98802432900000003 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880321+0000", 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3022e-05 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.116 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213409 5.10 5.e3d5e190 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.899439+0000", 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 349.18005905799998, 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98091395400000003, 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213409 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899439+0000", 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899439+0000", 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899438+0000", 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899443+0000", 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8749999999999999e-06 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899454+0000", 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1017e-05 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899461+0000", 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.391e-06 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899513+0000", 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2345000000000001e-05 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899522+0000", 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0590000000000003e-06 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880345+0000", 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98082366899999995 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880353+0000", 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0569999999999993e-06 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213416 5.10 5.10f69050 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.905431+0000", 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 349.174067057, 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97496326499999997, 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.121 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213416 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905431+0000", 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905431+0000", 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905430+0000", 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905438+0000", 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0849999999999997e-06 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905450+0000", 2024-09-21T10:14:22.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1790000000000001e-05 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905456+0000", 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.19e-06 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905491+0000", 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4449999999999997e-05 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905495+0000", 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6099999999999999e-06 2024-09-21T10:14:22.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880383+0000", 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97488764400000005 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880395+0000", 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.154e-05 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.125 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213453 5.10 5.4c1650d0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.939270+0000", 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 349.14022801599998, 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.94115652100000002, 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213453 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939270+0000", 2024-09-21T10:14:22.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939270+0000", 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939269+0000", 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939275+0000", 2024-09-21T10:14:22.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7909999999999999e-06 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939291+0000", 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5420000000000001e-05 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939296+0000", 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3290000000000001e-06 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939341+0000", 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4842000000000002e-05 2024-09-21T10:14:22.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.129 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.129 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.129 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939350+0000", 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8880000000000003e-06 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880416+0000", 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.94106558200000001 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880427+0000", 2024-09-21T10:14:22.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1365e-05 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213458 5.10 5.aa724650 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.947131+0000", 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 349.13236716, 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.93332198799999999, 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.131 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213458 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947131+0000", 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947131+0000", 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947130+0000", 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947136+0000", 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8250000000000001e-06 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947145+0000", 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2369999999999998e-06 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947152+0000", 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.64e-06 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947195+0000", 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2960000000000002e-05 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947203+0000", 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2269999999999993e-06 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880444+0000", 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.93324136300000005 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880453+0000", 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8200000000000003e-06 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213507 5.10 5.fbac80f0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.135 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.985956+0000", 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 349.09354267100002, 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.89453823399999999, 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213507 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985956+0000", 2024-09-21T10:14:22.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985956+0000", 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985955+0000", 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985960+0000", 2024-09-21T10:14:22.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3979999999999998e-06 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985975+0000", 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4622e-05 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985983+0000", 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9179999999999994e-06 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.986035+0000", 2024-09-21T10:14:22.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2018999999999998e-05 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.986042+0000", 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4869999999999996e-06 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880485+0000", 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.89444252199999996 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880494+0000", 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3680000000000008e-06 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213582 5.10 5.ab58c590 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.048306+0000", 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 349.03119221200001, 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.83221718, 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.140 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213582 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048306+0000", 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048306+0000", 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048305+0000", 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048310+0000", 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9150000000000004e-06 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048321+0000", 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0861e-05 2024-09-21T10:14:22.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048327+0000", 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6370000000000002e-06 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048387+0000", 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9551e-05 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048394+0000", 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.277e-06 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880518+0000", 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.83212388800000003 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880523+0000", 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2519999999999999e-06 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213626 5.10 5.4cd189b0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.084187+0000", 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 348.995311259, 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.79637013700000003, 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213626 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084187+0000", 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084187+0000", 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084186+0000", 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:22.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084191+0000", 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4729999999999999e-06 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084202+0000", 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0596e-05 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084209+0000", 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0360000000000001e-06 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084256+0000", 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7663000000000001e-05 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084264+0000", 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0220000000000004e-06 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880549+0000", 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.79628444099999995 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880557+0000", 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3350000000000007e-06 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213633 5.10 5.18cc6f50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.092236+0000", 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 348.987262493, 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.78834528699999995, 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.149 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213633 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092236+0000", 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092236+0000", 2024-09-21T10:14:22.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092234+0000", 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092243+0000", 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4630000000000007e-06 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092259+0000", 2024-09-21T10:14:22.151 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5245000000000001e-05 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092267+0000", 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6300000000000004e-06 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092311+0000", 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3951e-05 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092319+0000", 2024-09-21T10:14:22.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7640000000000005e-06 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880576+0000", 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.78825735200000002 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880581+0000", 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7190000000000001e-06 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213651 5.10 5.b09713b0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.108041+0000", 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 348.97145704899998, 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.77257185100000003, 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213651 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.154 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108041+0000", 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108041+0000", 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108040+0000", 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108045+0000", 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4750000000000004e-06 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108074+0000", 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9459999999999999e-05 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108086+0000", 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2035000000000001e-05 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108144+0000", 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8010000000000002e-05 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108153+0000", 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0830000000000004e-06 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880605+0000", 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.772452162 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.157 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880613+0000", 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4419999999999998e-06 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213671 5.10 5.775f8bb0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.127502+0000", 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 348.95199681899999, 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.75312580900000003, 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.158 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213671 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127502+0000", 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127502+0000", 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127501+0000", 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127505+0000", 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6770000000000003e-06 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127516+0000", 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0674e-05 2024-09-21T10:14:22.160 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127521+0000", 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8060000000000002e-06 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127557+0000", 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6684999999999999e-05 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127562+0000", 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9509999999999997e-06 2024-09-21T10:14:22.161 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880623+0000", 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.75306079199999998 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880627+0000", 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.245e-06 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.162 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213799 5.10 5.20d823f0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.279962+0000", 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 348.79953595799998, 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.60068750100000001, 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213799 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279962+0000", 2024-09-21T10:14:22.163 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.164 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.164 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.164 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.164 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279962+0000", 2024-09-21T10:14:22.165 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.165 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.165 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.165 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.165 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279961+0000", 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279967+0000", 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.665e-06 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279978+0000", 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1405e-05 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.166 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279985+0000", 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.652e-06 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.280054+0000", 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8783999999999997e-05 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.280062+0000", 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9780000000000005e-06 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.167 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880642+0000", 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.60057990299999997 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880650+0000", 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2979999999999992e-06 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213835 5.10 5.b1c1bcf0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.339730+0000", 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 348.73976807399998, 2024-09-21T10:14:22.168 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54093409000000003, 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213835 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339730+0000", 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.169 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339730+0000", 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339729+0000", 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339736+0000", 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9240000000000001e-06 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.170 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339757+0000", 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1168000000000001e-05 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339766+0000", 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7649999999999999e-06 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339824+0000", 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7893000000000003e-05 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.171 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339833+0000", 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.628e-06 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880660+0000", 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54082699199999995 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880664+0000", 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.233e-06 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.172 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213850 5.10 5.b9b1c390 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.365015+0000", 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 348.71448292899998, 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.515670454, 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213850 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.173 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365015+0000", 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365015+0000", 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365014+0000", 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:14:22.174 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365021+0000", 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6640000000000001e-06 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365033+0000", 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2131e-05 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365038+0000", 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5319999999999997e-06 2024-09-21T10:14:22.175 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365082+0000", 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3278999999999997e-05 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365090+0000", 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2150000000000001e-06 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880677+0000", 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.51558764300000004 2024-09-21T10:14:22.176 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880686+0000", 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4400000000000005e-06 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213854 5.10 5.6b4dc610 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.376078+0000", 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 348.70342065800003, 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.50462188200000002, 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.177 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213854 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376078+0000", 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.178 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376078+0000", 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376077+0000", 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376081+0000", 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2540000000000003e-06 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.179 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376090+0000", 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0000000000000002e-06 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376094+0000", 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.98e-06 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376130+0000", 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6358e-05 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:14:22.180 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376136+0000", 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6339999999999998e-06 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880695+0000", 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.50455955799999996 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880700+0000", 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1620000000000001e-06 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.181 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:133568 6.3 6:c24f731d:::benchmark_data_smithi032_171826_object3945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:11:55.785444+0000", 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 146.29405454299999, 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16183787999999999, 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 133568 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.785444+0000", 2024-09-21T10:14:22.182 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.785444+0000", 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.785442+0000", 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.785452+0000", 2024-09-21T10:14:22.183 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0577000000000001e-05 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.910964+0000", 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.12551145899999999 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.910995+0000", 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0917999999999998e-05 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.911269+0000", 2024-09-21T10:14:22.184 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00027455699999999998 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.911390+0000", 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120487 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.911498+0000", 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010770200000000001 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.920941+0000", 2024-09-21T10:14:22.185 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094430509999999992 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.920948+0000", 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9949999999999999e-06 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.947214+0000", 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.026266173 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:14:22.186 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.947250+0000", 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5828999999999999e-05 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:11:55.947282+0000", 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2091999999999997e-05 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:14:22.187 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:14:22.187 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:22.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:22.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:22.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:22.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:22.435 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:22.457 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:22.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:22.558 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:22.570 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:14:22.582 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:22.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:22.685 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:22.707 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:22.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:22.808 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:22.830 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:22.855 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:14:22.932 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:22.954 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:22.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:23.056 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:23.078 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:23.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:23.180 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:23.202 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:23.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:23.304 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:23.326 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:23.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:23.327 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:23.412 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:14:23.428 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:23.452 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:23.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:23.576 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:23.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:23.700 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:23.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:23.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:23 smithi032 ceph-mon[105748]: pgmap v698: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:23.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3376294913' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:23.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:23 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:23.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:23 smithi032 ceph-mon[101299]: pgmap v698: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:23.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3376294913' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:23.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:23 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:23 smithi045 ceph-mon[95976]: pgmap v698: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3376294913' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:23.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:23 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:23.802 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:23.825 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:23.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:23.927 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:23.949 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:24.051 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:24.073 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:24.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:24.175 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:24.199 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:24.301 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:24.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:24.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:24.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:24.449 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:24.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:24.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:24.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:24.677 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:24.700 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:24.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:24 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:14:24.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:24 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:14:24.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:24 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:24.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:24 smithi032 ceph-mon[101299]: osdmap e244: 8 total, 8 up, 7 in 2024-09-21T10:14:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:24 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:14:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:24 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:14:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:24 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:24.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:24 smithi032 ceph-mon[105748]: osdmap e244: 8 total, 8 up, 7 in 2024-09-21T10:14:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:24 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:14:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:24 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:14:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:24 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:24.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:24 smithi045 ceph-mon[95976]: osdmap e244: 8 total, 8 up, 7 in 2024-09-21T10:14:24.802 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:24.826 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:24.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:24.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:24.951 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:24.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:25.053 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:25.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:25.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:25.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:25.205 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:25.306 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:25.331 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:25.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:25.458 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:25.559 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:25.583 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:25.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:25.685 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:25.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:25 smithi032 ceph-mon[101299]: pgmap v700: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:25.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:25 smithi032 ceph-mon[105748]: pgmap v700: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:25.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:25.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:25.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:25 smithi045 ceph-mon[95976]: pgmap v700: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:25.811 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:25.838 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:25.939 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:25.965 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:26.066 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:26.090 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:26.191 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:26.217 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:26.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:26.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:26.343 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:26.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:26.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:26.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:26.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:26.594 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:26.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:26.696 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:26.718 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:26.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:26.815 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:26.815 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:26.820 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:26.821 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:14:26.821 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:26.846 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:26.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:26.848 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:14:26.848 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:26.866 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:14:26.866 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:26.882 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:14:26.882 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:26.899 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:14:26.899 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:26.915 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:14:26.916 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:26.932 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:14:26.933 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:26.948 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:26.949 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:14:26.949 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:26.971 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:26.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:27.072 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:27.098 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:27.199 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:27.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:27.323 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:27.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:27.432 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_2' created 2024-09-21T10:14:27.451 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:27.474 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:27.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:27.576 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:27.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:27.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:27.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:27.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:27 smithi032 ceph-mon[101299]: pgmap v701: 73 pgs: 73 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:27.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3398847894' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:14:27.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:27 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:14:27.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3404002402' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:27 smithi032 ceph-mon[105748]: pgmap v701: 73 pgs: 73 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3398847894' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:14:27.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:27 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:14:27.709 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3404002402' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:27.728 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:27 smithi045 ceph-mon[95976]: pgmap v701: 73 pgs: 73 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3398847894' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:14:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:27 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:14:27.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3404002402' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:27.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:27.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:14:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.853 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:27.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:27.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:27.981 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:27.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:28.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:28.106 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:28.208 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:28.230 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:28.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:28.331 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:28.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:28.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:28.456 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:28.480 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:28.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:28.581 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:28.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:28.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:28.706 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:28.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:28 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-21T10:14:28.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:28 smithi032 ceph-mon[105748]: osdmap e245: 8 total, 8 up, 7 in 2024-09-21T10:14:28.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:28 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-21T10:14:28.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:28 smithi032 ceph-mon[101299]: osdmap e245: 8 total, 8 up, 7 in 2024-09-21T10:14:28.730 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:28.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:28.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:28 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-21T10:14:28.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:28 smithi045 ceph-mon[95976]: osdmap e245: 8 total, 8 up, 7 in 2024-09-21T10:14:28.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:28.855 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:28.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:28.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:28.982 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:28.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:29.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:29.108 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:29.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:29.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-09-21T10:14:29.210 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:29.247 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:29.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:29.348 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:29.375 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:29.477 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:29.502 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:29.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:29.627 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:29.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:14:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:14:29] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.51.0" 2024-09-21T10:14:29.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:29 smithi032 ceph-mon[105748]: pgmap v703: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:29.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:29 smithi032 ceph-mon[105748]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:14:29.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:29 smithi032 ceph-mon[105748]: osdmap e246: 8 total, 8 up, 7 in 2024-09-21T10:14:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:29 smithi032 ceph-mon[101299]: pgmap v703: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:29.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:29 smithi032 ceph-mon[101299]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:14:29.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:29 smithi032 ceph-mon[101299]: osdmap e246: 8 total, 8 up, 7 in 2024-09-21T10:14:29.729 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:29.755 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:29 smithi045 ceph-mon[95976]: pgmap v703: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:29 smithi045 ceph-mon[95976]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:14:29.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:29 smithi045 ceph-mon[95976]: osdmap e246: 8 total, 8 up, 7 in 2024-09-21T10:14:29.857 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:29.882 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:29.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:29.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:30.006 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:30.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:30.108 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:30.134 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:30.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:30.236 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:30.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:30.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:30.360 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:30.382 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:30.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:30.484 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:30.492 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:14:30.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.507 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:30.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:30.608 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:30.636 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:30.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:30.704 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:30 smithi032 ceph-mon[101299]: osdmap e247: 8 total, 8 up, 7 in 2024-09-21T10:14:30.705 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:30 smithi032 ceph-mon[105748]: osdmap e247: 8 total, 8 up, 7 in 2024-09-21T10:14:30.737 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:30.759 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:30.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:30.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:30 smithi045 ceph-mon[95976]: osdmap e247: 8 total, 8 up, 7 in 2024-09-21T10:14:30.861 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:30.883 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:30.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:30.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:31.007 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:31.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:31.108 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:31.131 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:31.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:31.233 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:31.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:31.361 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:31.389 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:31.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:31.447 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:14:31.491 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:31.500 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:31.500 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:31.515 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:31.616 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:31.641 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:31.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:31.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:31 smithi032 ceph-mon[101299]: pgmap v706: 89 pgs: 4 unknown, 85 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:31.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:31 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/155854308' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:14:31.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:31 smithi032 ceph-mon[105748]: pgmap v706: 89 pgs: 4 unknown, 85 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:31.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:31 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/155854308' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:14:31.743 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:31.767 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:31.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:31.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:31 smithi045 ceph-mon[95976]: pgmap v706: 89 pgs: 4 unknown, 85 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:31.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:31 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/155854308' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:14:31.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:31.893 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:31.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:31.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:32.021 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:32.082 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:14:32.090 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:32.090 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:32.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:32.148 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:32.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:32.250 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:32.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:32.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:32.374 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:32.400 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:32.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:32.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:32.528 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:32.629 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:32.652 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:32.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:32.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:32 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/155854308' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:14:32.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:32 smithi032 ceph-mon[101299]: osdmap e248: 8 total, 8 up, 7 in 2024-09-21T10:14:32.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:32 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/155854308' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:14:32.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:32 smithi032 ceph-mon[105748]: osdmap e248: 8 total, 8 up, 7 in 2024-09-21T10:14:32.754 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:32.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:32.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:32.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:32 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/155854308' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:14:32.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:32 smithi045 ceph-mon[95976]: osdmap e248: 8 total, 8 up, 7 in 2024-09-21T10:14:32.879 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:32.901 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:32.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:33.003 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:33.030 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:33.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:33.131 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:33.153 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:33.255 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:33.281 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:33.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:33.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:33.405 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:33.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:33.507 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:33.530 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:33.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:33.631 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:33.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:33.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:33.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:33 smithi032 ceph-mon[101299]: pgmap v708: 89 pgs: 4 unknown, 85 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:33.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:33 smithi032 ceph-mon[101299]: osdmap e249: 8 total, 8 up, 7 in 2024-09-21T10:14:33.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:33 smithi032 ceph-mon[105748]: pgmap v708: 89 pgs: 4 unknown, 85 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:33.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:33 smithi032 ceph-mon[105748]: osdmap e249: 8 total, 8 up, 7 in 2024-09-21T10:14:33.759 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:33.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:33 smithi045 ceph-mon[95976]: pgmap v708: 89 pgs: 4 unknown, 85 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:33.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:33 smithi045 ceph-mon[95976]: osdmap e249: 8 total, 8 up, 7 in 2024-09-21T10:14:33.786 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:33.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:33.888 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:33.910 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:33.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:34.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:34.034 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:34.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:34.135 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:34.157 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:34.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:34.165 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:34.258 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:34.281 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:34.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:34.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:34.405 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:34.506 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:34.528 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:34.569 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:34.570 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:34.570 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:34.570 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:34.590 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:14:34.630 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:34.655 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:34.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:34.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:34.783 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:34.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:34.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:34.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:34.885 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:34.909 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:34.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:35.011 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:35.034 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:35.136 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:35.158 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:35.259 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:35.281 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:35.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:35.313 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:35.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:35.406 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:35.507 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:35.529 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:35.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:35.631 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:35.653 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:35.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:35.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:35 smithi032 ceph-mon[101299]: pgmap v710: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:35.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:35 smithi032 ceph-mon[105748]: pgmap v710: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:35.754 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:35.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:35.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:35.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:35 smithi045 ceph-mon[95976]: pgmap v710: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:35.877 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:35.904 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:35.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:36.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:36.028 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:36.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:36.130 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:36.157 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:36.258 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:36.284 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:36.386 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:36.412 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:36.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:36.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:36.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:36.685 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:36.713 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:36.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:36.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:36.841 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:36.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:36.943 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:36.970 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:36.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:37.071 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:37.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:37.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:37.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:37.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:37.324 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:37.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:37.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:37.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:37.479 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:37.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:37.580 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:37.604 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:37.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:37.706 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:37.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:37 smithi032 ceph-mon[101299]: pgmap v711: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:37.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:37 smithi032 ceph-mon[105748]: pgmap v711: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:37.728 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:37.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:37.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:37 smithi045 ceph-mon[95976]: pgmap v711: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:37.803 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:37.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:37.852 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:37.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:37.954 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:37.976 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:37.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:38.078 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:38.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:38.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:38.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:38.232 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:38.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:38.333 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:38.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:38.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:38.462 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:38.488 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:38.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:38.590 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:38.615 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:38.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:38.717 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:38.744 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:38.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:38.845 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:38.872 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:38.973 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:38.999 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:39.101 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:39.127 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:39.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:39.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:39.254 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:39.355 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:39.382 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:39.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:39.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:39.499 INFO:teuthology.orchestra.run.smithi032.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-09-21T10:14:39.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:39.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:39.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:39.635 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:39.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:39.700 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:14:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:14:39] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.51.0" 2024-09-21T10:14:39.701 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:39 smithi032 ceph-mon[101299]: pgmap v712: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:39.701 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2614881673' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:14:39.701 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:39 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:14:39.701 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:39 smithi032 ceph-mon[105748]: pgmap v712: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:39.701 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2614881673' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:14:39.701 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:39 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:14:39.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:39.762 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:39 smithi045 ceph-mon[95976]: pgmap v712: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:39 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2614881673' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:14:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:39 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:14:39.863 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:39.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:39.991 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:40.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:40.141 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:40.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:40.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.265 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:40.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:40.366 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:40.391 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:40.493 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:40.517 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:40.597 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:40 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:14:40.597 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:40 smithi032 ceph-mon[105748]: osdmap e250: 8 total, 8 up, 7 in 2024-09-21T10:14:40.597 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:40 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1804817012' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:40.597 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:40 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:14:40.597 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:40 smithi032 ceph-mon[101299]: osdmap e250: 8 total, 8 up, 7 in 2024-09-21T10:14:40.597 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:40 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1804817012' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:40.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:14:40.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:14:40.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:14:40.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:14:40.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:14:40.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:14:40.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:14:40.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:14:40.615 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:40.616 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:40.616 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:40.616 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:40.616 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:40.616 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:40.616 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:40.616 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:40.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:40.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:40.743 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.765 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:40.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:40 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:14:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:40 smithi045 ceph-mon[95976]: osdmap e250: 8 total, 8 up, 7 in 2024-09-21T10:14:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:40 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1804817012' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:40.866 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.889 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:40.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:40.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:41.012 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:41.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:41.114 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:41.141 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:41.242 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:41.269 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:41.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:41.370 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:41.392 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:41.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:41.494 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:41.521 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:41.623 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:41.650 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:41.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:41.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:41 smithi032 ceph-mon[105748]: pgmap v714: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:41.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:41 smithi032 ceph-mon[105748]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:14:41.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:41 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:14:41.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:41 smithi032 ceph-mon[101299]: pgmap v714: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:41.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:41 smithi032 ceph-mon[101299]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:14:41.675 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:41 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:14:41.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:41.774 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:41.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:41 smithi045 ceph-mon[95976]: pgmap v714: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:41 smithi045 ceph-mon[95976]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:14:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:41 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:14:41.875 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:41.902 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:41.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:42.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:42.026 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:42.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:42.128 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:42.154 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:42.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:42.256 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:42.282 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:42.384 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:42.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:42.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:42.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:42.512 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:14:42.536 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:42.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:42.638 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:42.664 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:42.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:42.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:42 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1517330456' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:42.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:42 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:42.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:42 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1517330456' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:42.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:42 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:42.766 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:42 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1517330456' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:42 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:42.791 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:42.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:42.893 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:42.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:42.983 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:14:42.983 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:14:42.983 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:14:42.983 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:14:42.984 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:14:43.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:43.042 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:43.144 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:43.165 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:43.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:43.267 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:43.294 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:43.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:43.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:14:43.396 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:43.422 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:43.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:43.523 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:43.547 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:43.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:43.648 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:43.653 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:43 smithi032 ceph-mon[105748]: pgmap v715: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:43.654 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:43 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:43.654 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:43 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:14:43.654 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:43 smithi032 ceph-mon[105748]: osdmap e251: 8 total, 8 up, 7 in 2024-09-21T10:14:43.654 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:43 smithi032 ceph-mon[101299]: pgmap v715: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:43.654 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:43 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:43.654 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:43 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:14:43.654 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:43 smithi032 ceph-mon[101299]: osdmap e251: 8 total, 8 up, 7 in 2024-09-21T10:14:43.670 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:43.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:43.772 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:43.777 DEBUG:teuthology.orchestra.run.smithi032:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-09-21T10:14:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:43 smithi045 ceph-mon[95976]: pgmap v715: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:43 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:43 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:14:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:43 smithi045 ceph-mon[95976]: osdmap e251: 8 total, 8 up, 7 in 2024-09-21T10:14:43.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.795 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:43.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:43.796 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:14:43.797 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:14:43.847 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:hints = 1 2024-09-21T10:14:43.847 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:14:43.848 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object prefix: benchmark_data_smithi032_200324 2024-09-21T10:14:43.848 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:14:43.848 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:14:43.896 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:43.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:43.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:44.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:44.073 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:44.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:44.175 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:44.216 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:44.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:44.303 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:14:44.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:44.389 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:44.491 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:44.509 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:44 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3805167216' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:14:44.515 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:14:44.524 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:44.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:44.625 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:44.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:44.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:44.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:44.772 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:44.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:44.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:44 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3805167216' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:14:44.848 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 1 16 2439 2423 151.419 151.438 0.00150437 0.00451963 2024-09-21T10:14:44.874 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:44.889 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:44 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3805167216' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:44.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:45.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:45.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:45.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:45.145 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:45.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:45.246 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:45.269 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:45.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:45.370 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:45.392 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:45.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:45.494 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:45.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:45.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:45.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:45.639 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:45.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:45 smithi032 ceph-mon[101299]: pgmap v717: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:45 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3805167216' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:14:45.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:45 smithi032 ceph-mon[101299]: osdmap e252: 8 total, 8 up, 7 in 2024-09-21T10:14:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:45 smithi032 ceph-mon[105748]: pgmap v717: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:45 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3805167216' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:14:45.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:45 smithi032 ceph-mon[105748]: osdmap e252: 8 total, 8 up, 7 in 2024-09-21T10:14:45.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:45.764 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:45.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:14:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:45 smithi045 ceph-mon[95976]: pgmap v717: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:14:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:45 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3805167216' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:14:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:45 smithi045 ceph-mon[95976]: osdmap e252: 8 total, 8 up, 7 in 2024-09-21T10:14:45.848 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 2 16 2439 2423 75.711 0 - 0.00451963 2024-09-21T10:14:45.865 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:45.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:45.888 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:45.896 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:45.989 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:46.012 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:46.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:46.114 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:46.136 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:46.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:46.238 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:46.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:46.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:46.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:46.385 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:46.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:46.487 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:46.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:46.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:46.611 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:46.634 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:46.735 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:46.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.759 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:46.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:46 smithi045 ceph-mon[95976]: osdmap e253: 8 total, 8 up, 7 in 2024-09-21T10:14:46.848 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 3 16 2605 2589 53.9321 5.1875 0.00440372 0.0185266 2024-09-21T10:14:46.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:46.895 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:46.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:46 smithi032 ceph-mon[101299]: osdmap e253: 8 total, 8 up, 7 in 2024-09-21T10:14:46.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:46 smithi032 ceph-mon[105748]: osdmap e253: 8 total, 8 up, 7 in 2024-09-21T10:14:46.997 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:47.030 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:47.132 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:47.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:47.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:47.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:47.302 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:14:47.315 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:47.419 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:47.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.461 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:47.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:47.562 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:47.588 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:47 smithi032 ceph-mon[105748]: pgmap v720: 89 pgs: 1 unknown, 12 peering, 76 active+clean; 88 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 221 op/s 2024-09-21T10:14:47.589 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:47 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-21T10:14:47.589 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:47 smithi032 ceph-mon[105748]: osdmap e254: 8 total, 8 up, 7 in 2024-09-21T10:14:47.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:47 smithi032 ceph-mon[101299]: pgmap v720: 89 pgs: 1 unknown, 12 peering, 76 active+clean; 88 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 221 op/s 2024-09-21T10:14:47.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:47 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-21T10:14:47.589 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:47 smithi032 ceph-mon[101299]: osdmap e254: 8 total, 8 up, 7 in 2024-09-21T10:14:47.601 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:47.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:47.702 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:47.725 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:47.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:47 smithi045 ceph-mon[95976]: pgmap v720: 89 pgs: 1 unknown, 12 peering, 76 active+clean; 88 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 221 op/s 2024-09-21T10:14:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:47 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-21T10:14:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:47 smithi045 ceph-mon[95976]: osdmap e254: 8 total, 8 up, 7 in 2024-09-21T10:14:47.827 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:47.848 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 4 16 5049 5033 78.6331 152.75 0.00120584 0.0119351 2024-09-21T10:14:47.850 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:47.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:47.952 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:47.974 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:47.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:48.003 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:48.076 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:48.098 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:48.200 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:48.223 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:48.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:48.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:48.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:48.448 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:48.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:48.572 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:48.598 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:48.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:48.699 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:48.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:48 smithi032 ceph-mon[105748]: osdmap e255: 8 total, 8 up, 7 in 2024-09-21T10:14:48.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:48.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:48.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:48 smithi032 ceph-mon[101299]: osdmap e255: 8 total, 8 up, 7 in 2024-09-21T10:14:48.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:48.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:48.721 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:14:48.722 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.722 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:48.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:48.738 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:14:48.739 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:48.755 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:14:48.755 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:48.771 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:14:48.771 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:48.788 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:14:48.788 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:48.804 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:14:48.804 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:48.821 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:14:48.822 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:48.823 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:48.838 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:14:48.838 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:48.845 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:48.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:48.848 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 5 16 5125 5109 63.8566 4.75 0.00128458 0.0147762 2024-09-21T10:14:48.947 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:48.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:48 smithi045 ceph-mon[95976]: osdmap e255: 8 total, 8 up, 7 in 2024-09-21T10:14:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:49.071 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:49.093 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:49.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:49.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.218 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:49.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:49.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:49.342 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:49.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:49.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:49.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:49.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:49.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:49.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.608 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:49.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:49.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:49 smithi032 ceph-mon[105748]: pgmap v723: 89 pgs: 1 active+recovering+undersized+remapped, 1 unknown, 12 peering, 75 active+clean; 159 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 39 MiB/s wr, 617 op/s; 130/7494 objects misplaced (1.735%); 400 KiB/s, 6 objects/s recovering 2024-09-21T10:14:49.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:49 smithi032 ceph-mon[105748]: osdmap e256: 8 total, 8 up, 7 in 2024-09-21T10:14:49.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:49 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/155501875' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:49.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:49 smithi032 ceph-mon[101299]: pgmap v723: 89 pgs: 1 active+recovering+undersized+remapped, 1 unknown, 12 peering, 75 active+clean; 159 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 39 MiB/s wr, 617 op/s; 130/7494 objects misplaced (1.735%); 400 KiB/s, 6 objects/s recovering 2024-09-21T10:14:49.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:49 smithi032 ceph-mon[101299]: osdmap e256: 8 total, 8 up, 7 in 2024-09-21T10:14:49.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:49 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/155501875' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:49.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:14:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:14:49] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.51.0" 2024-09-21T10:14:49.709 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:49.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:49.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:49.847 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:49.848 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 6 16 6163 6147 64.0254 64.875 0.00391139 0.0156095 2024-09-21T10:14:49.881 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:49.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:49.983 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:50.019 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:49 smithi045 ceph-mon[95976]: pgmap v723: 89 pgs: 1 active+recovering+undersized+remapped, 1 unknown, 12 peering, 75 active+clean; 159 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 39 MiB/s wr, 617 op/s; 130/7494 objects misplaced (1.735%); 400 KiB/s, 6 objects/s recovering 2024-09-21T10:14:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:49 smithi045 ceph-mon[95976]: osdmap e256: 8 total, 8 up, 7 in 2024-09-21T10:14:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:49 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/155501875' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:50.121 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:50.157 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:50.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:50.259 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:50.296 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:50.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:50.398 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:50.438 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:50.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:50.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:50.580 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:50.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:50.633 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:50 smithi032 ceph-mon[101299]: osdmap e257: 8 total, 8 up, 7 in 2024-09-21T10:14:50.634 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:50 smithi032 ceph-mon[105748]: osdmap e257: 8 total, 8 up, 7 in 2024-09-21T10:14:50.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:50.714 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:50.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:50.816 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:50.849 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 7 16 9662 9646 86.1172 218.688 0.0039659 0.0116055 2024-09-21T10:14:50.852 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:50.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:50.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:50.990 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:50.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:50 smithi045 ceph-mon[95976]: osdmap e257: 8 total, 8 up, 7 in 2024-09-21T10:14:51.091 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:51.129 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:51.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:51.231 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:51.273 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:51.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:51.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:51.415 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:51.517 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.541 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:51.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:51.642 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:51.678 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:51.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:51.809 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:51 smithi032 ceph-mon[101299]: pgmap v726: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 329 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s; 130/15642 objects misplaced (0.831%); 9.7 MiB/s, 157 objects/s recovering 2024-09-21T10:14:51.809 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:51 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-21T10:14:51.809 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:51 smithi032 ceph-mon[105748]: pgmap v726: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 329 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s; 130/15642 objects misplaced (0.831%); 9.7 MiB/s, 157 objects/s recovering 2024-09-21T10:14:51.809 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:51 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-21T10:14:51.810 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:51.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:51.849 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 8 16 12672 12656 98.866 188.125 0.0114645 0.0100987 2024-09-21T10:14:51.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:51.992 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:51.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:51 smithi045 ceph-mon[95976]: pgmap v726: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 329 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s; 130/15642 objects misplaced (0.831%); 9.7 MiB/s, 157 objects/s recovering 2024-09-21T10:14:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:51 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-21T10:14:52.093 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:52.139 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:52.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:52.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:14:52.241 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:52.245 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:52.245 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:52.296 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:52.397 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:52.426 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:52.528 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:52.550 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:52.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:52.557 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:14:52.651 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:52.674 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:52.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:52 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:14:52.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:52 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1834936387' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:14:52.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:52 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:14:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:52 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:14:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:52 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1834936387' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:14:52.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:52 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:14:52.776 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:52.816 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:52.849 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 9 16 14607 14591 101.313 120.938 0.00761964 0.0098613 2024-09-21T10:14:52.918 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:52.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.965 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:52.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:53.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:52 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:14:53.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:52 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1834936387' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:14:53.037 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:52 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:14:53.066 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:53.104 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:53.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:53.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:53.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.245 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:53.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:53.323 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:14:53.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.412 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:53.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:53.513 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:53.521 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:53.547 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:53.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:53.649 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:53.688 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:53.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:53.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:53.829 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:53.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:53.849 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 10 16 17396 17380 108.611 174.312 0.00512471 0.00920049 2024-09-21T10:14:53.931 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:53 smithi032 ceph-mon[105748]: pgmap v727: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 329 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 41 MiB/s wr, 660 op/s; 130/15642 objects misplaced (0.831%); 6.7 MiB/s, 108 objects/s recovering 2024-09-21T10:14:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:53 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:14:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:53 smithi032 ceph-mon[105748]: osdmap e258: 8 total, 8 up, 7 in 2024-09-21T10:14:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:53 smithi032 ceph-mon[101299]: pgmap v727: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 329 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 41 MiB/s wr, 660 op/s; 130/15642 objects misplaced (0.831%); 6.7 MiB/s, 108 objects/s recovering 2024-09-21T10:14:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:53 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:14:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:53 smithi032 ceph-mon[101299]: osdmap e258: 8 total, 8 up, 7 in 2024-09-21T10:14:53.969 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:53.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:53 smithi045 ceph-mon[95976]: pgmap v727: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 329 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 41 MiB/s wr, 660 op/s; 130/15642 objects misplaced (0.831%); 6.7 MiB/s, 108 objects/s recovering 2024-09-21T10:14:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:53 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:14:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:53 smithi045 ceph-mon[95976]: osdmap e258: 8 total, 8 up, 7 in 2024-09-21T10:14:54.071 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:54.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:54.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:54.211 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:54.250 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:54.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:54.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 7 1 2024-09-21T10:14:54.352 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:54.388 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:54.490 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:54.512 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:54.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:54.614 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:54.663 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:54.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:54.765 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:54.801 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:54.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:54.849 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 11 16 19771 19755 112.23 148.438 0.00414078 0.00890543 2024-09-21T10:14:54.904 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:54.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:55.035 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:55.071 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:55.173 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:55.213 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:55.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:55.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:55.352 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:55.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:55.455 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:55.489 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:55.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:55.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:55.637 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:55.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:55.739 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:55.783 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:55.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:55 smithi045 ceph-mon[95976]: pgmap v729: 89 pgs: 89 active+clean; 578 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 7.6 MiB/s, 122 objects/s recovering 2024-09-21T10:14:55.795 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:55 smithi032 ceph-mon[105748]: pgmap v729: 89 pgs: 89 active+clean; 578 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 7.6 MiB/s, 122 objects/s recovering 2024-09-21T10:14:55.796 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:55 smithi032 ceph-mon[101299]: pgmap v729: 89 pgs: 89 active+clean; 578 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 7.6 MiB/s, 122 objects/s recovering 2024-09-21T10:14:55.849 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 12 16 22254 22238 115.809 155.188 0.00521197 0.00862918 2024-09-21T10:14:55.886 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:55.923 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:55.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:56.025 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:56.071 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:56.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:56.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:56.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.229 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:56.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:56.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:56.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.365 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:56.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:56.466 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:56.504 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:56.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:56.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:56.643 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:56.729 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:56.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:56.817 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:56.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:56.850 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 13 16 24681 24665 118.565 151.688 0.00680685 0.0084281 2024-09-21T10:14:56.919 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:56.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:57.063 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:57.103 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:57.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:57.205 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:57.242 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:57.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:57.382 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:57.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:57.484 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:57.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.523 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:57.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:57.624 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:57.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:57.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:57.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:57.799 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:57.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:57.850 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 14 16 27142 27126 121.081 153.812 0.0112402 0.00825293 2024-09-21T10:14:57.891 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:57 smithi032 ceph-mon[105748]: pgmap v730: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 122 MiB/s wr, 1.96k op/s; 5.8 MiB/s, 94 objects/s recovering 2024-09-21T10:14:57.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:57 smithi032 ceph-mon[101299]: pgmap v730: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 122 MiB/s wr, 1.96k op/s; 5.8 MiB/s, 94 objects/s recovering 2024-09-21T10:14:57.901 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:57.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.957 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:57.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:57 smithi045 ceph-mon[95976]: pgmap v730: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 122 MiB/s wr, 1.96k op/s; 5.8 MiB/s, 94 objects/s recovering 2024-09-21T10:14:58.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:58.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:58.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:58.197 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:58.236 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:58.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:58.337 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:58.374 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:58.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:58.476 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:58.513 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:58.615 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:58.643 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:58.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:58.745 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:58.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.783 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:58.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:58.850 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 15 16 29056 29040 120.984 119.625 0.00460572 0.00825977 2024-09-21T10:14:58.884 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:58.907 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:58.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:59.009 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:59.044 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:59.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:59.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:59.162 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:58 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:59.163 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:58 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:59.163 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:58 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:59.163 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:58 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:59.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.186 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:59.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:58 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:58 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:14:59.286 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:59.315 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:59.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:59.417 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:59.438 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:59.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:59.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:59.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:59.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:59.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.701 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:14:59.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:59.802 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:59.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.831 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:59.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:59.850 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 16 16 30768 30752 120.109 107 0.00527529 0.00832033 2024-09-21T10:14:59.851 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:14:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:14:59] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.51.0" 2024-09-21T10:14:59.931 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:59.967 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:14:59.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:00.069 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:00.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:00.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:00.194 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:14:59 smithi032 ceph-mon[101299]: pgmap v731: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s; 4.0 MiB/s, 64 objects/s recovering 2024-09-21T10:15:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:14:59 smithi032 ceph-mon[105748]: pgmap v731: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s; 4.0 MiB/s, 64 objects/s recovering 2024-09-21T10:15:00.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:14:59 smithi045 ceph-mon[95976]: pgmap v731: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s; 4.0 MiB/s, 64 objects/s recovering 2024-09-21T10:15:00.330 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:00.363 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:00.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:00.465 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:00.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.501 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:00.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:00.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:00.624 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:00.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:00.726 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:00.761 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:00.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:00.850 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 17 16 32360 32344 118.896 99.5 0.00670876 0.00840551 2024-09-21T10:15:00.864 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:00.897 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:00.998 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:01.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:01.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:01.165 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:01.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:00 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3258156056' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:00 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3258156056' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:01.267 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:01.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:00 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3258156056' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:01.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:01.288 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:01.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:01.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:01.412 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:01.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:01.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:01.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:01.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:01.652 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:01.693 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:01.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:01.758 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:15:01.795 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.831 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:01.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:01.850 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 18 16 34127 34111 118.426 110.438 0.00533822 0.00843904 2024-09-21T10:15:01.932 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:01.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.973 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:01.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:02.074 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:02.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:02.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:01 smithi032 ceph-mon[105748]: pgmap v732: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 150 MiB/s wr, 2.40k op/s; 832 KiB/s, 12 objects/s recovering 2024-09-21T10:15:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:01 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:01 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:01 smithi032 ceph-mon[105748]: osdmap e259: 8 total, 8 up, 7 in 2024-09-21T10:15:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:01 smithi032 ceph-mon[101299]: pgmap v732: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 150 MiB/s wr, 2.40k op/s; 832 KiB/s, 12 objects/s recovering 2024-09-21T10:15:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:01 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:01 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:01 smithi032 ceph-mon[101299]: osdmap e259: 8 total, 8 up, 7 in 2024-09-21T10:15:02.212 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:02.230 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:02.244 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:02.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:02.245 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:01 smithi045 ceph-mon[95976]: pgmap v732: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 150 MiB/s wr, 2.40k op/s; 832 KiB/s, 12 objects/s recovering 2024-09-21T10:15:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:01 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:01 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:01 smithi045 ceph-mon[95976]: osdmap e259: 8 total, 8 up, 7 in 2024-09-21T10:15:02.346 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:02.378 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:02.481 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:02.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:02.618 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:02.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:02.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:02.757 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:02.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:02.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:02.850 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 19 16 36321 36305 119.409 137.125 0.0055076 0.00836768 2024-09-21T10:15:02.899 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:02.935 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:03.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:03.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:03.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:03.175 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:03.236 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:03.337 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:03.385 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:03.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:03.487 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:03.519 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:03.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:03.621 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:03.655 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:03.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:03.757 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:03.806 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:03.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:15:03.849575+0000 min lat: 0.00117819 max lat: 2.28911 avg lat: 0.00833109 2024-09-21T10:15:03.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:15:03.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 20 16 38408 38392 119.96 130.438 0.00368878 0.00833109 2024-09-21T10:15:03.907 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:03.939 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:03.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:04.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:04.083 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:04.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:04.184 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:03 smithi032 ceph-mon[105748]: pgmap v734: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-21T10:15:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:03 smithi032 ceph-mon[101299]: pgmap v734: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-21T10:15:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:04.218 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:03 smithi045 ceph-mon[95976]: pgmap v734: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-21T10:15:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:04.320 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:04.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:04.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:04.462 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:04.495 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:04.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:04.597 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:04.632 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:04.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:04.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:04.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 21 16 40477 40461 120.405 129.312 0.00509293 0.00829877 2024-09-21T10:15:04.878 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.917 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:04.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:05.018 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:05.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.056 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:05.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:05.157 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:05.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:15:05.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:05.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:05.289 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:05.316 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:05.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:05.418 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:05.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.451 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:05.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:05.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:05.577 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:05.679 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:05.716 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:05.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:05.817 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:05.847 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:05.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:05.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 22 16 42268 42252 120.02 111.938 0.0120374 0.00832682 2024-09-21T10:15:05.949 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:05.994 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:06.096 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:06.131 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:06.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:05 smithi032 ceph-mon[105748]: pgmap v735: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-21T10:15:06.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:05 smithi032 ceph-mon[101299]: pgmap v735: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-21T10:15:06.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:06.264 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:06.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:06.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:05 smithi045 ceph-mon[95976]: pgmap v735: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-21T10:15:06.366 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:06.401 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:06.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:06.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:06.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.549 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:06.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:06.650 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:06.687 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:06.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:06.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:06.828 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:06.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:06.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 23 16 44020 44004 119.562 109.5 0.00521234 0.00835905 2024-09-21T10:15:06.930 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:06.938 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.7 to 1 (10000) 2024-09-21T10:15:06.962 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:06.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:07.064 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:07.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.112 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:07.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:07.126 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/192304287' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:15:07.127 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:06 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:15:07.127 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/192304287' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:15:07.127 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:06 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:15:07.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.258 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:07.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/192304287' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:15:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:06 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:15:07.326 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:15:07.327 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:07.359 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:07.365 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:15:07.366 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:07.400 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:15:07.400 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:07.402 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:07.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:07.437 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:15:07.437 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:15:07.474 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:15:07.475 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:07.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:07.512 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:15:07.512 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:07.547 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:07.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:07.562 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:15:07.562 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:07.633 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:15:07.633 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:07.662 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:07.704 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:07.806 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:07.847 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:07.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 24 16 45763 45747 119.118 108.938 0.00976017 0.00838946 2024-09-21T10:15:07.948 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:07.985 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:08.087 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:08.129 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:08.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:07 smithi032 ceph-mon[105748]: pgmap v736: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-21T10:15:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:15:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:07 smithi032 ceph-mon[105748]: osdmap e260: 8 total, 8 up, 7 in 2024-09-21T10:15:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:07 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2322331431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:07 smithi032 ceph-mon[101299]: pgmap v736: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-21T10:15:08.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:15:08.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:07 smithi032 ceph-mon[101299]: osdmap e260: 8 total, 8 up, 7 in 2024-09-21T10:15:08.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:07 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2322331431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:08.231 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:08.268 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:07 smithi045 ceph-mon[95976]: pgmap v736: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-21T10:15:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:15:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:07 smithi045 ceph-mon[95976]: osdmap e260: 8 total, 8 up, 7 in 2024-09-21T10:15:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:07 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2322331431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:08.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:15:08.370 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:08.406 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:08.508 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:08.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.554 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:08.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:08.655 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:08.691 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:08.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:47466 7.b 7:d408beb4:::benchmark_data_smithi032_200324_object47465:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:15:08.777456+0000", 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0010009330000000001, 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001032245, 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:15:08.782 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 47466 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.777456+0000", 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.777456+0000", 2024-09-21T10:15:08.783 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.777457+0000", 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.584e-06 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.777481+0000", 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3842e-05 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.777494+0000", 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2968e-05 2024-09-21T10:15:08.784 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.777502+0000", 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5360000000000004e-06 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.777569+0000", 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7343999999999998e-05 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.777696+0000", 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012656500000000001 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:08.785 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:47465 7.7 e260/245 7:e6bd59b4:::benchmark_data_smithi032_200324_object47464:head v 260'2960, mlcod=260'2958)", 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:15:08.776572+0000", 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0018849439999999999, 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0019803109999999998, 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.786 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.776572+0000", 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.776572+0000", 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.776573+0000", 2024-09-21T10:15:08.787 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1100000000000004e-07 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.776609+0000", 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6217000000000002e-05 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.776620+0000", 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1022999999999999e-05 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.788 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.776626+0000", 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0499999999999997e-06 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.776703+0000", 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7088000000000005e-05 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:15:08.789 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:08.776723+0000", 2024-09-21T10:15:08.790 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9984999999999999e-05 2024-09-21T10:15:08.790 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:08.790 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:15:08.790 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:08.790 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:08.790 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:15:08.790 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 2 2024-09-21T10:15:08.790 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:15:08.792 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:08.824 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:08.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:08.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 25 16 47578 47562 118.889 113.438 0.00688969 0.0084057 2024-09-21T10:15:08.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:08.969 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:08.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:09.071 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:09.110 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:09.212 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:09.248 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:09.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:09.429 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:09.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:09.530 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:09.564 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:09.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:09.666 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:09.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:15:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:15:09] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.51.0" 2024-09-21T10:15:09.708 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:09.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:09.810 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:09.848 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:09.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:09.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 26 16 49434 49418 118.778 116 0.00754505 0.00841445 2024-09-21T10:15:09.950 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:09.985 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:09.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:10.058 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:09 smithi032 ceph-mon[105748]: pgmap v738: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:15:10.059 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:09 smithi032 ceph-mon[101299]: pgmap v738: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:15:10.087 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:10.113 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:15:10.128 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:10.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.135 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:10.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:10.235 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:10.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:10.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:09 smithi045 ceph-mon[95976]: pgmap v738: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:15:10.386 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:10.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.420 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:10.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:10.521 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:10.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:10.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:10.665 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:10.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.705 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:10.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:10.806 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:10.842 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:10.852 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 27 16 51062 51046 118.147 101.75 0.00930852 0.00845942 2024-09-21T10:15:10.943 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.981 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:10.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:11.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:11.121 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:11.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:11.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:11.266 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:11.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:11.353 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:15:11.368 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:11.401 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:11.505 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:11.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:11.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:11.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:11.687 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:11.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:11.789 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:11.841 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:11.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:11.851 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 28 16 52766 52750 117.73 106.5 0.014136 0.00848835 2024-09-21T10:15:11.944 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:11.983 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:12.001 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:11 smithi032 ceph-mon[105748]: pgmap v739: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-09-21T10:15:12.001 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:11 smithi032 ceph-mon[101299]: pgmap v739: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-09-21T10:15:12.085 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:12.115 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:12.117 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:12.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:12.219 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:12.267 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:12.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:11 smithi045 ceph-mon[95976]: pgmap v739: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-09-21T10:15:12.369 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:12.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.403 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:12.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:12.504 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.552 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:12.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:12.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:12.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:12.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:12.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:12.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:12.852 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 29 16 54354 54338 117.093 99.25 0.00937254 0.00853471 2024-09-21T10:15:12.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:12.977 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:12.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:13.082 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:13.120 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:13.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:13.260 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:13.361 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:13.403 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:13.504 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:13.543 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:13.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:13.645 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:13.686 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:13.705 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:13.711 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:13.788 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:13.828 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:13.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:13.852 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 30 16 55690 55674 115.973 83.5 0.0102339 0.00861722 2024-09-21T10:15:13.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:13.958 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:15:13.963 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:14.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:14.103 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:14.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:14.205 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:14.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:13 smithi032 ceph-mon[101299]: pgmap v740: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-21T10:15:14.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3515307084' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:14.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:13 smithi032 ceph-mon[105748]: pgmap v740: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-21T10:15:14.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3515307084' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:14.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.244 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:14.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:13 smithi045 ceph-mon[95976]: pgmap v740: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-21T10:15:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3515307084' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:14.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:14.401 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:14.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:14.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:14.549 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:14.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:14.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.691 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:14.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:14.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:14.831 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:14.852 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 31 16 57148 57132 115.171 91.125 0.0113958 0.00867768 2024-09-21T10:15:14.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:14.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.971 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:14.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:15.072 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:15.113 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:15.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:15.146 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:14 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:15.146 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:14 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3515307084' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:15.146 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:14 smithi032 ceph-mon[105748]: osdmap e261: 8 total, 8 up, 7 in 2024-09-21T10:15:15.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:14 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:15.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:14 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3515307084' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:15.147 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:14 smithi032 ceph-mon[101299]: osdmap e261: 8 total, 8 up, 7 in 2024-09-21T10:15:15.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:15.253 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:15.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:14 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:14 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3515307084' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:14 smithi045 ceph-mon[95976]: osdmap e261: 8 total, 8 up, 7 in 2024-09-21T10:15:15.355 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:15.391 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:15.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:15.493 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:15.535 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:15.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:15.637 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:15.677 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:15.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:15.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:15.822 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:15.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:15.852 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 32 16 58728 58712 114.658 98.75 0.0148697 0.00871655 2024-09-21T10:15:15.924 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:15.960 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:16.062 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:16.100 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:16.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:16.202 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:16.242 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:16.243 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:15 smithi032 ceph-mon[105748]: pgmap v742: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-21T10:15:16.243 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:15 smithi032 ceph-mon[101299]: pgmap v742: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-21T10:15:16.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:15 smithi045 ceph-mon[95976]: pgmap v742: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-21T10:15:16.345 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:16.373 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:16.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:16.474 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:16.509 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:16.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:16.610 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:16.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.650 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:16.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:16.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:16.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:16.852 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 33 16 59909 59893 113.42 73.8125 0.00312922 0.00879511 2024-09-21T10:15:16.880 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:16.916 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:17.018 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:17.060 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:17.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:17.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:17.187 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:15:17.199 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:17.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:17.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.366 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:17.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:17.467 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:17.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.490 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:17.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:17.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:17.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:17.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:17.734 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:17.763 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:17.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:17.852 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 34 16 61208 61192 112.471 81.1875 0.00588189 0.00888601 2024-09-21T10:15:17.865 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:17.899 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:17.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:18.001 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:18.023 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:18.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:18.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:18.162 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:18.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:18.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:17 smithi045 ceph-mon[95976]: pgmap v743: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-09-21T10:15:18.301 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:18.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:18.439 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:18.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:17 smithi032 ceph-mon[101299]: pgmap v743: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-09-21T10:15:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:17 smithi032 ceph-mon[105748]: pgmap v743: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-09-21T10:15:18.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:18.584 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:18.686 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.729 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:18.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:18.830 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:18.852 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 35 16 62688 62672 111.901 92.5 0.00799948 0.00893116 2024-09-21T10:15:18.863 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:18.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:18.965 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:18.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:18.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:19.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:19.136 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:19.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:19.238 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:19.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.278 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:19.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:19.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:19.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:19.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:19.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:19.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:19.553 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:19.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:19.655 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:19.692 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:19.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:19.794 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:19.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:19.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:19.852 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 36 16 64276 64260 111.549 99.25 0.00471384 0.00895842 2024-09-21T10:15:19.919 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:15:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:15:19] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.51.0" 2024-09-21T10:15:19.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:19.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.994 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:19.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:20.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:20.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:20.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:20.184 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:19 smithi032 ceph-mon[105748]: pgmap v744: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-21T10:15:20.184 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]} : dispatch 2024-09-21T10:15:20.184 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]} : dispatch 2024-09-21T10:15:20.184 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]} : dispatch 2024-09-21T10:15:20.184 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:19 smithi032 ceph-mon[101299]: pgmap v744: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-21T10:15:20.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]} : dispatch 2024-09-21T10:15:20.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]} : dispatch 2024-09-21T10:15:20.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]} : dispatch 2024-09-21T10:15:20.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:20.265 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:20.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:19 smithi045 ceph-mon[95976]: pgmap v744: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-21T10:15:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]} : dispatch 2024-09-21T10:15:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]} : dispatch 2024-09-21T10:15:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]} : dispatch 2024-09-21T10:15:20.367 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:20.400 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:20.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:20.501 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:20.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:20.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:20.636 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:20.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:20.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:20.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:20.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.823 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:20.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:20.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 37 16 65698 65682 110.936 88.875 0.0104501 0.00900823 2024-09-21T10:15:20.924 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:20.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.957 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:20.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:21.032 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-21T10:15:21.032 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-21T10:15:21.032 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2024-09-21T10:15:21.032 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[105748]: osdmap e262: 8 total, 8 up, 7 in 2024-09-21T10:15:21.032 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3130388258' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:15:21.032 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:15:21.032 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-21T10:15:21.033 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-21T10:15:21.033 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2024-09-21T10:15:21.033 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[101299]: osdmap e262: 8 total, 8 up, 7 in 2024-09-21T10:15:21.033 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3130388258' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:15:21.033 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:21 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:15:21.034 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:15:21.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:21.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:21.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:21.202 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:21.243 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-21T10:15:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-21T10:15:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2024-09-21T10:15:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:21 smithi045 ceph-mon[95976]: osdmap e262: 8 total, 8 up, 7 in 2024-09-21T10:15:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:21 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3130388258' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:15:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:21 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:15:21.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:21.387 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:21.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:21.489 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:21.528 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:21.630 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:21.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:21.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:21.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:21.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.801 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:21.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:21.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 38 16 67136 67120 110.381 89.875 0.0107847 0.00905475 2024-09-21T10:15:21.902 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:21.935 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:21.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:22.037 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:22.070 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:22.172 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:22 smithi032 ceph-mon[101299]: pgmap v746: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T10:15:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:22 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:15:22.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:22 smithi032 ceph-mon[101299]: osdmap e263: 8 total, 8 up, 7 in 2024-09-21T10:15:22.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:22 smithi032 ceph-mon[105748]: pgmap v746: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T10:15:22.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:22 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:15:22.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:22 smithi032 ceph-mon[105748]: osdmap e263: 8 total, 8 up, 7 in 2024-09-21T10:15:22.208 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:22.285 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:15:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=cleanup t=2024-09-21T10:15:22.127068932Z level=info msg="Completed cleanup jobs" duration=3.217357ms 2024-09-21T10:15:22.285 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:15:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=plugins.update.checker t=2024-09-21T10:15:22.264654674Z level=info msg="Update check succeeded" duration=66.848793ms 2024-09-21T10:15:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:22 smithi045 ceph-mon[95976]: pgmap v746: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T10:15:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:22 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:15:22.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:22 smithi045 ceph-mon[95976]: osdmap e263: 8 total, 8 up, 7 in 2024-09-21T10:15:22.310 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:22.339 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:15:22.339 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:15:22.339 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:15:22.339 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:15:22.339 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:15:22.348 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:22.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:22.450 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:22.485 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:22.587 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.670 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:22.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:22.771 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:22.812 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:22.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:22.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 39 16 68485 68469 109.713 84.3125 0.0137287 0.00910941 2024-09-21T10:15:22.914 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.964 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:22.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:23.065 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:23.101 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:23.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:23.242 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:23.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:23.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:23.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:23.483 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:23.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.523 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:23.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:23.624 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:23.645 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:23.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:23.747 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:23.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.786 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:23.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:23.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:15:23.851891+0000 min lat: 0.00117819 max lat: 2.28911 avg lat: 0.00916693 2024-09-21T10:15:23.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:15:23.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 40 16 69805 69789 109.032 82.5 0.00472381 0.00916693 2024-09-21T10:15:23.887 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:23.909 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:23.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:24.010 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:24.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:24.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:24.149 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:24.186 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:24 smithi032 ceph-mon[101299]: pgmap v748: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:15:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:24 smithi032 ceph-mon[105748]: pgmap v748: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:15:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:24 smithi045 ceph-mon[95976]: pgmap v748: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:15:24.287 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:24.323 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:24.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:24.424 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:24.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.464 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:24.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:24.565 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:24.587 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:24.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:24.688 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:24.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:24.826 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:24.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 41 16 71084 71068 108.322 79.9375 0.0102315 0.0092265 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:24.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:24.965 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:25.003 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:25.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:25.104 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:25.126 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:25.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:25.228 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:25.268 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:25.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:25.370 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:25.407 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:25.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:25.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:25.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:25.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:25.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:25.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:25.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:25.781 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:25.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:25.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:25.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 42 16 72281 72265 107.524 74.8125 0.0081398 0.00929539 2024-09-21T10:15:25.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:25.945 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:25.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:26.047 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:26.083 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:26.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:26.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:26.207 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:26.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:26 smithi045 ceph-mon[95976]: pgmap v749: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:15:26.308 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:26.340 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:26.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:26.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:26 smithi032 ceph-mon[101299]: pgmap v749: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:15:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:26 smithi032 ceph-mon[105748]: pgmap v749: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:15:26.477 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:26.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:26.579 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:26.619 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:26.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:26.721 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:26.759 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:26.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:26.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 43 16 73432 73416 106.697 71.9375 0.00986063 0.00936743 2024-09-21T10:15:26.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:26.906 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:26.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:27.007 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:27.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:27.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:27.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.181 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:27.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:27.282 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:27.321 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:27.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:27.422 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:27.457 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:27.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:27.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:27.596 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:27.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:27.698 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:27.731 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:27.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:27.833 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:27.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 44 16 74690 74674 106.059 78.625 0.0147835 0.009424 2024-09-21T10:15:27.874 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:27.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:27.976 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:28.016 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:28.117 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:28.139 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:15:28.162 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:28.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:28.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:28.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.319 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:28.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:28.328 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:28 smithi032 ceph-mon[101299]: pgmap v750: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-21T10:15:28.328 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:28 smithi032 ceph-mon[105748]: pgmap v750: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-21T10:15:28.420 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:28.421 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:15:28.422 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:28.454 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:28.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:28.455 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:15:28.456 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:28.504 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:15:28.504 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:28.528 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:15:28.528 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:15:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:28 smithi045 ceph-mon[95976]: pgmap v750: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-21T10:15:28.545 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:15:28.546 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:15:28.556 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:28.578 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:28.579 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:15:28.580 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:28.581 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:28.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:28.683 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:28.814 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:28.844 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:28.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 45 16 75914 75898 105.402 76.5 0.0131869 0.0094825 2024-09-21T10:15:28.946 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:28.982 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:28.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:29.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:29.134 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:29.236 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:29.281 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:29.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:29.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:29.424 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:29.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:29.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:29 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2983241337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:29.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:29 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2983241337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:29.526 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:29 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2983241337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.566 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:29.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:29.667 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:29.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.702 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:29.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:29.760 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:15:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:15:29] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.51.0" 2024-09-21T10:15:29.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:29.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.833 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:29.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:29.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 46 16 77233 77217 104.902 82.4375 0.00682553 0.00952686 2024-09-21T10:15:29.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:29.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.990 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:29.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:30.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:30.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:30.251 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:30.279 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:30.382 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:30.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:30.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:30.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:30 smithi032 ceph-mon[105748]: pgmap v751: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:15:30.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:30.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:30 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:30.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:30 smithi032 ceph-mon[101299]: pgmap v751: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:15:30.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:30.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:30 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:30.483 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:30.517 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:30.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:30 smithi045 ceph-mon[95976]: pgmap v751: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:15:30.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:30.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:30 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:30.545 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:30.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:30.648 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:30.684 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:30.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:30.785 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:30.820 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:30.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:30.853 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 47 16 78380 78364 104.195 71.6875 0.0110512 0.00959093 2024-09-21T10:15:30.922 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:30.965 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:30.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:31.050 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:31.068 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.099 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:31.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:31.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:15:31.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:15:31.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:31.177 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:15:31.177 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:15:31.177 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:31.200 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:31.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.253 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:31.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:31.355 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:31.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.393 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:31.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:31.494 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:31.532 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:31.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:31.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:15:31.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:15:31.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:31.633 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:31.664 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:31.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:31.766 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:31.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:31.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:31.854 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 48 16 79563 79547 103.565 73.9375 0.00821389 0.00965001 2024-09-21T10:15:31.896 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.933 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:31.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:32.034 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:32.066 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:32.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:32.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:32.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:32.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:32.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:32.352 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:32.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:32.454 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:32 smithi032 ceph-mon[105748]: pgmap v752: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-09-21T10:15:32.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:32 smithi032 ceph-mon[101299]: pgmap v752: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-09-21T10:15:32.495 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:32 smithi045 ceph-mon[95976]: pgmap v752: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-09-21T10:15:32.597 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:32.635 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:32.737 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:32.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:32.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:32.858 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 49 16 80827 80811 103.063 79 0.0127271 0.00969789 2024-09-21T10:15:32.902 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:32.939 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:32.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:33.041 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:33.083 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:33.185 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:33.220 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:33.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:33.322 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:33.365 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:33.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:33.467 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:33.500 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:33.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:33.644 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:33.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:33.746 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:33.783 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:15:33.791 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:33.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:33.856 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 50 16 82163 82147 102.667 83.5 0.0156765 0.00973461 2024-09-21T10:15:33.893 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:33.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:33.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:34.052 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:34.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.082 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:34.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:34.183 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:34 smithi032 ceph-mon[101299]: pgmap v753: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-21T10:15:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:34 smithi032 ceph-mon[105748]: pgmap v753: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-21T10:15:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:34.217 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:34.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:34.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:34.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:34.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:34.504 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:34 smithi045 ceph-mon[95976]: pgmap v753: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-21T10:15:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:34.606 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:34.648 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:34.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:34.750 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.782 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:34.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:34.856 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 51 16 83372 83356 102.135 75.5625 0.0181738 0.00978582 2024-09-21T10:15:34.883 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:34.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:34.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:35.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:35.069 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:35.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:35.171 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:35.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.203 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:35.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:35.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:35.348 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:35.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:35.449 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:35.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:35.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:35.582 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:35.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.619 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:35.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:35.658 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:35.698 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:35.698 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:35.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:35.750 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:35.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:35.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:35.856 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 52 16 84519 84503 101.55 71.6875 0.00431663 0.00984052 2024-09-21T10:15:35.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:35.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:35.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:36.034 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:36.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:36.092 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:15:36.136 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:36.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:36.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:36.269 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:36.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:36.406 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:36.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.450 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:36.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:36 smithi032 ceph-mon[101299]: pgmap v754: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:15:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:36 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1516069822' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:36 smithi032 ceph-mon[105748]: pgmap v754: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:15:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:36 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1516069822' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:36 smithi045 ceph-mon[95976]: pgmap v754: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-21T10:15:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:36 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1516069822' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:36.551 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:36.587 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:36.638 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:15:36.639 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:15:36.689 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:36.725 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:36.827 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:36.856 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 53 16 85751 85735 101.087 77 0.0101375 0.00988734 2024-09-21T10:15:36.859 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:36.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:36.961 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:36.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.994 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:36.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:37.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:37.132 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:37.233 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:37.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.271 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:37.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:37.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:37.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:37.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:37 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:15:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:37 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:15:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:37 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1516069822' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:37 smithi032 ceph-mon[105748]: osdmap e264: 8 total, 8 up, 7 in 2024-09-21T10:15:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:37 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:15:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:37 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:15:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:37 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1516069822' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:37 smithi032 ceph-mon[101299]: osdmap e264: 8 total, 8 up, 7 in 2024-09-21T10:15:37.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:37 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:15:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:37 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:15:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:37 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1516069822' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:37 smithi045 ceph-mon[95976]: osdmap e264: 8 total, 8 up, 7 in 2024-09-21T10:15:37.555 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:37.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:37.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:37.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:37.801 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.836 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:37.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:37.856 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 54 16 87004 86988 100.665 78.3125 0.0134564 0.009929 2024-09-21T10:15:37.937 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:37.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.983 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:37.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:38.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:15:38.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:38.146 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:38.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:38.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:38.297 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:38.341 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:38 smithi032 ceph-mon[101299]: pgmap v756: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:15:38.341 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:38 smithi032 ceph-mon[101299]: from='client.55006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:38.341 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:38 smithi032 ceph-mon[105748]: pgmap v756: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:15:38.342 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:38 smithi032 ceph-mon[105748]: from='client.55006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:38.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:38.448 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:38 smithi045 ceph-mon[95976]: pgmap v756: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:15:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:38 smithi045 ceph-mon[95976]: from='client.55006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:38.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:38.594 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:38.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:38.696 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:38.737 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:38.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:38.839 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:38.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 55 16 88235 88219 100.233 76.9375 0.0082713 0.00997172 2024-09-21T10:15:38.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:38.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:38.984 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:39.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:39.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:39.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:39.160 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:39.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:39.261 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:39.299 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:39.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:39.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:39.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:39.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:39.550 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:39.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.584 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:39.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:39.587 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:15:39.683 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":756,"stamp":"2024-09-21T10:15:36.371522+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5400565733,"num_objects":82361,"num_object_clones":0,"num_object_copies":247083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82361,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":83456,"num_write_kb":5284353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":927,"num_bytes_recovered":65475136,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83515,"ondisk_log_size":83515,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18459096,"kb_used_data":15724256,"kb_used_omap":27,"kb_used_meta":2734756,"kb_avail":637756968,"statfs":{"total":671965249536,"available":653063135232,"internally_reserved":0,"allocated":16101638144,"data_stored":16093017515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28144,"internal_metadata":2800390672},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":807403520,"num_objects":12320,"num_object_clones":0,"num_object_copies":36960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12320,"num_write_kb":788480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002899"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":400,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.030508+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:15:21.030508+0000","last_peered":"2024-09-21T10:15:21.030508+0000","last_clean":"2024-09-21T10:15:21.030508+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:15:21.030508+0000","last_undegraded":"2024-09-21T10:15:21.030508+0000","last_fullsized":"2024-09-21T10:15:21.030508+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":232,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.014555+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:15:20.014555+0000","last_peered":"2024-09-21T10:15:20.014555+0000","last_clean":"2024-09-21T10:15:20.014555+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:15:20.014555+0000","last_undegraded":"2024-09-21T10:15:20.014555+0000","last_fullsized":"2024-09-21T10:15:20.014555+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":21,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.018255+0000","last_change":"2024-09-21T10:14:45.529072+0000","last_active":"2024-09-21T10:15:20.018255+0000","last_peered":"2024-09-21T10:15:20.018255+0000","last_clean":"2024-09-21T10:15:20.018255+0000","last_became_active":"2024-09-21T10:14:45.528270+0000","last_became_peered":"2024-09-21T10:14:45.528270+0000","last_unstale":"2024-09-21T10:15:20.018255+0000","last_undegraded":"2024-09-21T10:15:20.018255+0000","last_fullsized":"2024-09-21T10:15:20.018255+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:00.326719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.014503+0000","last_change":"2024-09-21T10:13:09.583319+0000","last_active":"2024-09-21T10:15:20.014503+0000","last_peered":"2024-09-21T10:15:20.014503+0000","last_clean":"2024-09-21T10:15:20.014503+0000","last_became_active":"2024-09-21T10:13:09.582953+0000","last_became_peered":"2024-09-21T10:13:09.582953+0000","last_unstale":"2024-09-21T10:15:20.014503+0000","last_undegraded":"2024-09-21T10:15:20.014503+0000","last_fullsized":"2024-09-21T10:15:20.014503+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:13.630031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":227,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.036140+0000","last_change":"2024-09-21T10:09:35.958207+0000","last_active":"2024-09-21T10:15:21.036140+0000","last_peered":"2024-09-21T10:15:21.036140+0000","last_clean":"2024-09-21T10:15:21.036140+0000","last_became_active":"2024-09-21T10:09:35.957860+0000","last_became_peered":"2024-09-21T10:09:35.957860+0000","last_unstale":"2024-09-21T10:15:21.036140+0000","last_undegraded":"2024-09-21T10:15:21.036140+0000","last_fullsized":"2024-09-21T10:15:21.036140+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:37.767930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":215,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.014880+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:15:20.014880+0000","last_peered":"2024-09-21T10:15:20.014880+0000","last_clean":"2024-09-21T10:15:20.014880+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:15:20.014880+0000","last_undegraded":"2024-09-21T10:15:20.014880+0000","last_fullsized":"2024-09-21T10:15:20.014880+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":406,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.045326+0000","last_change":"2024-09-21T10:14:20.761360+0000","last_active":"2024-09-21T10:15:21.045326+0000","last_peered":"2024-09-21T10:15:21.045326+0000","last_clean":"2024-09-21T10:15:21.045326+0000","last_became_active":"2024-09-21T10:14:20.760941+0000","last_became_peered":"2024-09-21T10:14:20.760941+0000","last_unstale":"2024-09-21T10:15:21.045326+0000","last_undegraded":"2024-09-21T10:15:21.045326+0000","last_fullsized":"2024-09-21T10:15:21.045326+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:27.842256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":280,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.013929+0000","last_change":"2024-09-21T10:14:45.537300+0000","last_active":"2024-09-21T10:15:20.013929+0000","last_peered":"2024-09-21T10:15:20.013929+0000","last_clean":"2024-09-21T10:15:20.013929+0000","last_became_active":"2024-09-21T10:14:45.527447+0000","last_became_peered":"2024-09-21T10:14:45.527447+0000","last_unstale":"2024-09-21T10:15:20.013929+0000","last_undegraded":"2024-09-21T10:15:20.013929+0000","last_fullsized":"2024-09-21T10:15:20.013929+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:33.652169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"263'5167","reported_seq":5501,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.802059+0000","last_change":"2024-09-21T10:14:49.554195+0000","last_active":"2024-09-21T10:15:34.802059+0000","last_peered":"2024-09-21T10:15:34.802059+0000","last_clean":"2024-09-21T10:15:34.802059+0000","last_became_active":"2024-09-21T10:14:49.551887+0000","last_became_peered":"2024-09-21T10:14:49.551887+0000","last_unstale":"2024-09-21T10:15:34.802059+0000","last_undegraded":"2024-09-21T10:15:34.802059+0000","last_fullsized":"2024-09-21T10:15:34.802059+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5167,"log_dups_size":0,"ondisk_log_size":5167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:26.931226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338624512,"num_objects":5167,"num_object_clones":0,"num_object_copies":15501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5167,"num_write_kb":330688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":9043968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":271,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.014880+0000","last_change":"2024-09-21T10:13:09.586457+0000","last_active":"2024-09-21T10:15:20.014880+0000","last_peered":"2024-09-21T10:15:20.014880+0000","last_clean":"2024-09-21T10:15:20.014880+0000","last_became_active":"2024-09-21T10:13:09.574624+0000","last_became_peered":"2024-09-21T10:13:09.574624+0000","last_unstale":"2024-09-21T10:15:20.014880+0000","last_undegraded":"2024-09-21T10:15:20.014880+0000","last_fullsized":"2024-09-21T10:15:20.014880+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:39:00.778086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":21,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.017680+0000","last_change":"2024-09-21T10:14:45.530298+0000","last_active":"2024-09-21T10:15:20.017680+0000","last_peered":"2024-09-21T10:15:20.017680+0000","last_clean":"2024-09-21T10:15:20.017680+0000","last_became_active":"2024-09-21T10:14:45.529912+0000","last_became_peered":"2024-09-21T10:14:45.529912+0000","last_unstale":"2024-09-21T10:15:20.017680+0000","last_undegraded":"2024-09-21T10:15:20.017680+0000","last_fullsized":"2024-09-21T10:15:20.017680+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:50.567684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"263'4964","reported_seq":5331,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:32.571064+0000","last_change":"2024-09-21T10:14:49.553650+0000","last_active":"2024-09-21T10:15:32.571064+0000","last_peered":"2024-09-21T10:15:32.571064+0000","last_clean":"2024-09-21T10:15:32.571064+0000","last_became_active":"2024-09-21T10:14:49.552007+0000","last_became_peered":"2024-09-21T10:14:49.552007+0000","last_unstale":"2024-09-21T10:15:32.571064+0000","last_undegraded":"2024-09-21T10:15:32.571064+0000","last_fullsized":"2024-09-21T10:15:32.571064+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":4964,"log_dups_size":0,"ondisk_log_size":4964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:43:27.336777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325320704,"num_objects":4964,"num_object_clones":0,"num_object_copies":14892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4964,"num_write_kb":317696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":298,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.015524+0000","last_change":"2024-09-21T10:13:09.596445+0000","last_active":"2024-09-21T10:15:20.015524+0000","last_peered":"2024-09-21T10:15:20.015524+0000","last_clean":"2024-09-21T10:15:20.015524+0000","last_became_active":"2024-09-21T10:13:09.582665+0000","last_became_peered":"2024-09-21T10:13:09.582665+0000","last_unstale":"2024-09-21T10:15:20.015524+0000","last_undegraded":"2024-09-21T10:15:20.015524+0000","last_fullsized":"2024-09-21T10:15:20.015524+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:23:47.510351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":263,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.017144+0000","last_change":"2024-09-21T10:13:09.578819+0000","last_active":"2024-09-21T10:15:20.017144+0000","last_peered":"2024-09-21T10:15:20.017144+0000","last_clean":"2024-09-21T10:15:20.017144+0000","last_became_active":"2024-09-21T10:13:09.576707+0000","last_became_peered":"2024-09-21T10:13:09.576707+0000","last_unstale":"2024-09-21T10:15:20.017144+0000","last_undegraded":"2024-09-21T10:15:20.017144+0000","last_fullsized":"2024-09-21T10:15:20.017144+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:19.175846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"263'5243","reported_seq":5297,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.780028+0000","last_change":"2024-09-21T10:14:32.771740+0000","last_active":"2024-09-21T10:15:34.780028+0000","last_peered":"2024-09-21T10:15:34.780028+0000","last_clean":"2024-09-21T10:15:34.780028+0000","last_became_active":"2024-09-21T10:14:32.771298+0000","last_became_peered":"2024-09-21T10:14:32.771298+0000","last_unstale":"2024-09-21T10:15:34.780028+0000","last_undegraded":"2024-09-21T10:15:34.780028+0000","last_fullsized":"2024-09-21T10:15:34.780028+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5243,"log_dups_size":0,"ondisk_log_size":5243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:27.287728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343605248,"num_objects":5243,"num_object_clones":0,"num_object_copies":15729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5243,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":243,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.014797+0000","last_change":"2024-09-21T10:14:45.533171+0000","last_active":"2024-09-21T10:15:20.014797+0000","last_peered":"2024-09-21T10:15:20.014797+0000","last_clean":"2024-09-21T10:15:20.014797+0000","last_became_active":"2024-09-21T10:14:45.527709+0000","last_became_peered":"2024-09-21T10:14:45.527709+0000","last_unstale":"2024-09-21T10:15:20.014797+0000","last_undegraded":"2024-09-21T10:15:20.014797+0000","last_fullsized":"2024-09-21T10:15:20.014797+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:33.989738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":248,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.013893+0000","last_change":"2024-09-21T10:13:09.578358+0000","last_active":"2024-09-21T10:15:20.013893+0000","last_peered":"2024-09-21T10:15:20.013893+0000","last_clean":"2024-09-21T10:15:20.013893+0000","last_became_active":"2024-09-21T10:13:09.577273+0000","last_became_peered":"2024-09-21T10:13:09.577273+0000","last_unstale":"2024-09-21T10:15:20.013893+0000","last_undegraded":"2024-09-21T10:15:20.013893+0000","last_fullsized":"2024-09-21T10:15:20.013893+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:17.971932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"263'5122","reported_seq":5168,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:33.797552+0000","last_change":"2024-09-21T10:14:28.438058+0000","last_active":"2024-09-21T10:15:33.797552+0000","last_peered":"2024-09-21T10:15:33.797552+0000","last_clean":"2024-09-21T10:15:33.797552+0000","last_became_active":"2024-09-21T10:14:28.437772+0000","last_became_peered":"2024-09-21T10:14:28.437772+0000","last_unstale":"2024-09-21T10:15:33.797552+0000","last_undegraded":"2024-09-21T10:15:33.797552+0000","last_fullsized":"2024-09-21T10:15:33.797552+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5122,"log_dups_size":0,"ondisk_log_size":5122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:14.437061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335675392,"num_objects":5122,"num_object_clones":0,"num_object_copies":15366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5122,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":231,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.018181+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:15:20.018181+0000","last_peered":"2024-09-21T10:15:20.018181+0000","last_clean":"2024-09-21T10:15:20.018181+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:15:20.018181+0000","last_undegraded":"2024-09-21T10:15:20.018181+0000","last_fullsized":"2024-09-21T10:15:20.018181+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":359,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.033235+0000","last_change":"2024-09-21T10:13:56.569595+0000","last_active":"2024-09-21T10:15:21.033235+0000","last_peered":"2024-09-21T10:15:21.033235+0000","last_clean":"2024-09-21T10:15:21.033235+0000","last_became_active":"2024-09-21T10:13:56.567665+0000","last_became_peered":"2024-09-21T10:13:56.567665+0000","last_unstale":"2024-09-21T10:15:21.033235+0000","last_undegraded":"2024-09-21T10:15:21.033235+0000","last_fullsized":"2024-09-21T10:15:21.033235+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:12:35.947882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"263'5077","reported_seq":5124,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:32.578243+0000","last_change":"2024-09-21T10:14:28.441282+0000","last_active":"2024-09-21T10:15:32.578243+0000","last_peered":"2024-09-21T10:15:32.578243+0000","last_clean":"2024-09-21T10:15:32.578243+0000","last_became_active":"2024-09-21T10:14:28.440767+0000","last_became_peered":"2024-09-21T10:14:28.440767+0000","last_unstale":"2024-09-21T10:15:32.578243+0000","last_undegraded":"2024-09-21T10:15:32.578243+0000","last_fullsized":"2024-09-21T10:15:32.578243+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5077,"log_dups_size":0,"ondisk_log_size":5077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:56:18.651559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332726272,"num_objects":5077,"num_object_clones":0,"num_object_copies":15231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5077,"num_write_kb":324928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":417,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.031398+0000","last_change":"2024-09-21T10:13:09.594302+0000","last_active":"2024-09-21T10:15:21.031398+0000","last_peered":"2024-09-21T10:15:21.031398+0000","last_clean":"2024-09-21T10:15:21.031398+0000","last_became_active":"2024-09-21T10:13:09.576948+0000","last_became_peered":"2024-09-21T10:13:09.576948+0000","last_unstale":"2024-09-21T10:15:21.031398+0000","last_undegraded":"2024-09-21T10:15:21.031398+0000","last_fullsized":"2024-09-21T10:15:21.031398+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:10.360060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":230,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.016954+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:15:20.016954+0000","last_peered":"2024-09-21T10:15:20.016954+0000","last_clean":"2024-09-21T10:15:20.016954+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:15:20.016954+0000","last_undegraded":"2024-09-21T10:15:20.016954+0000","last_fullsized":"2024-09-21T10:15:20.016954+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"263'5179","reported_seq":5571,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:33.776582+0000","last_change":"2024-09-21T10:14:49.553783+0000","last_active":"2024-09-21T10:15:33.776582+0000","last_peered":"2024-09-21T10:15:33.776582+0000","last_clean":"2024-09-21T10:15:33.776582+0000","last_became_active":"2024-09-21T10:14:49.552803+0000","last_became_peered":"2024-09-21T10:14:49.552803+0000","last_unstale":"2024-09-21T10:15:33.776582+0000","last_undegraded":"2024-09-21T10:15:33.776582+0000","last_fullsized":"2024-09-21T10:15:33.776582+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5179,"log_dups_size":0,"ondisk_log_size":5179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:56:20.507993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339410944,"num_objects":5179,"num_object_clones":0,"num_object_copies":15537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5179,"num_write_kb":331456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":166,"num_bytes_recovered":10878976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":249,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.014087+0000","last_change":"2024-09-21T10:13:09.578151+0000","last_active":"2024-09-21T10:15:20.014087+0000","last_peered":"2024-09-21T10:15:20.014087+0000","last_clean":"2024-09-21T10:15:20.014087+0000","last_became_active":"2024-09-21T10:13:09.577233+0000","last_became_peered":"2024-09-21T10:13:09.577233+0000","last_unstale":"2024-09-21T10:15:20.014087+0000","last_undegraded":"2024-09-21T10:15:20.014087+0000","last_fullsized":"2024-09-21T10:15:20.014087+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:00.627590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":25,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.017433+0000","last_change":"2024-09-21T10:14:32.771000+0000","last_active":"2024-09-21T10:15:20.017433+0000","last_peered":"2024-09-21T10:15:20.017433+0000","last_clean":"2024-09-21T10:15:20.017433+0000","last_became_active":"2024-09-21T10:14:32.770562+0000","last_became_peered":"2024-09-21T10:14:32.770562+0000","last_unstale":"2024-09-21T10:15:20.017433+0000","last_undegraded":"2024-09-21T10:15:20.017433+0000","last_fullsized":"2024-09-21T10:15:20.017433+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:47.956518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"263'5175","reported_seq":5201,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.641094+0000","last_change":"2024-09-21T10:14:28.441368+0000","last_active":"2024-09-21T10:15:34.641094+0000","last_peered":"2024-09-21T10:15:34.641094+0000","last_clean":"2024-09-21T10:15:34.641094+0000","last_became_active":"2024-09-21T10:14:28.441092+0000","last_became_peered":"2024-09-21T10:14:28.441092+0000","last_unstale":"2024-09-21T10:15:34.641094+0000","last_undegraded":"2024-09-21T10:15:34.641094+0000","last_fullsized":"2024-09-21T10:15:34.641094+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5175,"log_dups_size":0,"ondisk_log_size":5175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:28.336965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339148800,"num_objects":5175,"num_object_clones":0,"num_object_copies":15525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5175,"num_write_kb":331200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":231,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.017299+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:15:20.017299+0000","last_peered":"2024-09-21T10:15:20.017299+0000","last_clean":"2024-09-21T10:15:20.017299+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:15:20.017299+0000","last_undegraded":"2024-09-21T10:15:20.017299+0000","last_fullsized":"2024-09-21T10:15:20.017299+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":248,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.017232+0000","last_change":"2024-09-21T10:13:09.579444+0000","last_active":"2024-09-21T10:15:20.017232+0000","last_peered":"2024-09-21T10:15:20.017232+0000","last_clean":"2024-09-21T10:15:20.017232+0000","last_became_active":"2024-09-21T10:13:09.579162+0000","last_became_peered":"2024-09-21T10:13:09.579162+0000","last_unstale":"2024-09-21T10:15:20.017232+0000","last_undegraded":"2024-09-21T10:15:20.017232+0000","last_fullsized":"2024-09-21T10:15:20.017232+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.051244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"263'5130","reported_seq":5157,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.612054+0000","last_change":"2024-09-21T10:14:28.439503+0000","last_active":"2024-09-21T10:15:34.612054+0000","last_peered":"2024-09-21T10:15:34.612054+0000","last_clean":"2024-09-21T10:15:34.612054+0000","last_became_active":"2024-09-21T10:14:28.438413+0000","last_became_peered":"2024-09-21T10:14:28.438413+0000","last_unstale":"2024-09-21T10:15:34.612054+0000","last_undegraded":"2024-09-21T10:15:34.612054+0000","last_fullsized":"2024-09-21T10:15:34.612054+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5130,"log_dups_size":0,"ondisk_log_size":5130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:24.386364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336199680,"num_objects":5130,"num_object_clones":0,"num_object_copies":15390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5130,"num_write_kb":328320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":401,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.030616+0000","last_change":"2024-09-21T10:13:09.582151+0000","last_active":"2024-09-21T10:15:21.030616+0000","last_peered":"2024-09-21T10:15:21.030616+0000","last_clean":"2024-09-21T10:15:21.030616+0000","last_became_active":"2024-09-21T10:13:09.580224+0000","last_became_peered":"2024-09-21T10:13:09.580224+0000","last_unstale":"2024-09-21T10:15:21.030616+0000","last_undegraded":"2024-09-21T10:15:21.030616+0000","last_fullsized":"2024-09-21T10:15:21.030616+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:15.490461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":107,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.035216+0000","last_change":"2024-09-21T10:13:09.586643+0000","last_active":"2024-09-21T10:15:21.035216+0000","last_peered":"2024-09-21T10:15:21.035216+0000","last_clean":"2024-09-21T10:15:21.035216+0000","last_became_active":"2024-09-21T10:13:09.586283+0000","last_became_peered":"2024-09-21T10:13:09.586283+0000","last_unstale":"2024-09-21T10:15:21.035216+0000","last_undegraded":"2024-09-21T10:15:21.035216+0000","last_fullsized":"2024-09-21T10:15:21.035216+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:20.002373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"263'5245","reported_seq":5602,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.649503+0000","last_change":"2024-09-21T10:14:49.554445+0000","last_active":"2024-09-21T10:15:34.649503+0000","last_peered":"2024-09-21T10:15:34.649503+0000","last_clean":"2024-09-21T10:15:34.649503+0000","last_became_active":"2024-09-21T10:14:49.552633+0000","last_became_peered":"2024-09-21T10:14:49.552633+0000","last_unstale":"2024-09-21T10:15:34.649503+0000","last_undegraded":"2024-09-21T10:15:34.649503+0000","last_fullsized":"2024-09-21T10:15:34.649503+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5245,"log_dups_size":0,"ondisk_log_size":5245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:42.174531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343736320,"num_objects":5245,"num_object_clones":0,"num_object_copies":15735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5245,"num_write_kb":335680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":632,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.014613+0000","last_change":"2024-09-21T10:13:12.573143+0000","last_active":"2024-09-21T10:15:20.014613+0000","last_peered":"2024-09-21T10:15:20.014613+0000","last_clean":"2024-09-21T10:15:20.014613+0000","last_became_active":"2024-09-21T10:13:12.571751+0000","last_became_peered":"2024-09-21T10:13:12.571751+0000","last_unstale":"2024-09-21T10:15:20.014613+0000","last_undegraded":"2024-09-21T10:15:20.014613+0000","last_fullsized":"2024-09-21T10:15:20.014613+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:28.804119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":581,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.018089+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:15:20.018089+0000","last_peered":"2024-09-21T10:15:20.018089+0000","last_clean":"2024-09-21T10:15:20.018089+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:15:20.018089+0000","last_undegraded":"2024-09-21T10:15:20.018089+0000","last_fullsized":"2024-09-21T10:15:20.018089+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":348,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.030121+0000","last_change":"2024-09-21T10:14:45.527731+0000","last_active":"2024-09-21T10:15:21.030121+0000","last_peered":"2024-09-21T10:15:21.030121+0000","last_clean":"2024-09-21T10:15:21.030121+0000","last_became_active":"2024-09-21T10:14:45.527330+0000","last_became_peered":"2024-09-21T10:14:45.527330+0000","last_unstale":"2024-09-21T10:15:21.030121+0000","last_undegraded":"2024-09-21T10:15:21.030121+0000","last_fullsized":"2024-09-21T10:15:21.030121+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:03:48.411604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":216,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.032474+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:15:21.032474+0000","last_peered":"2024-09-21T10:15:21.032474+0000","last_clean":"2024-09-21T10:15:21.032474+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:15:21.032474+0000","last_undegraded":"2024-09-21T10:15:21.032474+0000","last_fullsized":"2024-09-21T10:15:21.032474+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"263'5199","reported_seq":5226,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.677460+0000","last_change":"2024-09-21T10:14:28.440396+0000","last_active":"2024-09-21T10:15:34.677460+0000","last_peered":"2024-09-21T10:15:34.677460+0000","last_clean":"2024-09-21T10:15:34.677460+0000","last_became_active":"2024-09-21T10:14:28.440137+0000","last_became_peered":"2024-09-21T10:14:28.440137+0000","last_unstale":"2024-09-21T10:15:34.677460+0000","last_undegraded":"2024-09-21T10:15:34.677460+0000","last_fullsized":"2024-09-21T10:15:34.677460+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5199,"log_dups_size":0,"ondisk_log_size":5199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:33:10.456362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340721664,"num_objects":5199,"num_object_clones":0,"num_object_copies":15597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5199,"num_write_kb":332736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":856,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.029885+0000","last_change":"2024-09-21T10:13:59.583227+0000","last_active":"2024-09-21T10:15:21.029885+0000","last_peered":"2024-09-21T10:15:21.029885+0000","last_clean":"2024-09-21T10:15:21.029885+0000","last_became_active":"2024-09-21T10:13:59.582623+0000","last_became_peered":"2024-09-21T10:13:59.582623+0000","last_unstale":"2024-09-21T10:15:21.029885+0000","last_undegraded":"2024-09-21T10:15:21.029885+0000","last_fullsized":"2024-09-21T10:15:21.029885+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:35.235896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":25,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.016984+0000","last_change":"2024-09-21T10:14:32.461889+0000","last_active":"2024-09-21T10:15:20.016984+0000","last_peered":"2024-09-21T10:15:20.016984+0000","last_clean":"2024-09-21T10:15:20.016984+0000","last_became_active":"2024-09-21T10:14:32.461551+0000","last_became_peered":"2024-09-21T10:14:32.461551+0000","last_unstale":"2024-09-21T10:15:20.016984+0000","last_undegraded":"2024-09-21T10:15:20.016984+0000","last_fullsized":"2024-09-21T10:15:20.016984+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:30:45.261092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":107,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.032554+0000","last_change":"2024-09-21T10:13:09.580568+0000","last_active":"2024-09-21T10:15:21.032554+0000","last_peered":"2024-09-21T10:15:21.032554+0000","last_clean":"2024-09-21T10:15:21.032554+0000","last_became_active":"2024-09-21T10:13:09.579053+0000","last_became_peered":"2024-09-21T10:13:09.579053+0000","last_unstale":"2024-09-21T10:15:21.032554+0000","last_undegraded":"2024-09-21T10:15:21.032554+0000","last_fullsized":"2024-09-21T10:15:21.032554+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:39.206612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"263'5206","reported_seq":5231,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.597920+0000","last_change":"2024-09-21T10:14:32.642821+0000","last_active":"2024-09-21T10:15:34.597920+0000","last_peered":"2024-09-21T10:15:34.597920+0000","last_clean":"2024-09-21T10:15:34.597920+0000","last_became_active":"2024-09-21T10:14:32.642404+0000","last_became_peered":"2024-09-21T10:14:32.642404+0000","last_unstale":"2024-09-21T10:15:34.597920+0000","last_undegraded":"2024-09-21T10:15:34.597920+0000","last_fullsized":"2024-09-21T10:15:34.597920+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5206,"log_dups_size":0,"ondisk_log_size":5206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:02.051331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341180416,"num_objects":5206,"num_object_clones":0,"num_object_copies":15618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5206,"num_write_kb":333184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":916,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.037425+0000","last_change":"2024-09-21T10:13:59.590250+0000","last_active":"2024-09-21T10:15:21.037425+0000","last_peered":"2024-09-21T10:15:21.037425+0000","last_clean":"2024-09-21T10:15:21.037425+0000","last_became_active":"2024-09-21T10:13:59.587283+0000","last_became_peered":"2024-09-21T10:13:59.587283+0000","last_unstale":"2024-09-21T10:15:21.037425+0000","last_undegraded":"2024-09-21T10:15:21.037425+0000","last_fullsized":"2024-09-21T10:15:21.037425+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:14.118934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":107,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.032137+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:15:21.032137+0000","last_peered":"2024-09-21T10:15:21.032137+0000","last_clean":"2024-09-21T10:15:21.032137+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:15:21.032137+0000","last_undegraded":"2024-09-21T10:15:21.032137+0000","last_fullsized":"2024-09-21T10:15:21.032137+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":107,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.035261+0000","last_change":"2024-09-21T10:13:09.578776+0000","last_active":"2024-09-21T10:15:21.035261+0000","last_peered":"2024-09-21T10:15:21.035261+0000","last_clean":"2024-09-21T10:15:21.035261+0000","last_became_active":"2024-09-21T10:13:09.578462+0000","last_became_peered":"2024-09-21T10:13:09.578462+0000","last_unstale":"2024-09-21T10:15:21.035261+0000","last_undegraded":"2024-09-21T10:15:21.035261+0000","last_fullsized":"2024-09-21T10:15:21.035261+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:19.035767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"263'5165","reported_seq":5192,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.643964+0000","last_change":"2024-09-21T10:14:28.440111+0000","last_active":"2024-09-21T10:15:34.643964+0000","last_peered":"2024-09-21T10:15:34.643964+0000","last_clean":"2024-09-21T10:15:34.643964+0000","last_became_active":"2024-09-21T10:14:28.439733+0000","last_became_peered":"2024-09-21T10:14:28.439733+0000","last_unstale":"2024-09-21T10:15:34.643964+0000","last_undegraded":"2024-09-21T10:15:34.643964+0000","last_fullsized":"2024-09-21T10:15:34.643964+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5165,"log_dups_size":0,"ondisk_log_size":5165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:17.534207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338493440,"num_objects":5165,"num_object_clones":0,"num_object_copies":15495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5165,"num_write_kb":330560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1055,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.018230+0000","last_change":"2024-09-21T10:13:12.560467+0000","last_active":"2024-09-21T10:15:20.018230+0000","last_peered":"2024-09-21T10:15:20.018230+0000","last_clean":"2024-09-21T10:15:20.018230+0000","last_became_active":"2024-09-21T10:13:12.559882+0000","last_became_peered":"2024-09-21T10:13:12.559882+0000","last_unstale":"2024-09-21T10:15:20.018230+0000","last_undegraded":"2024-09-21T10:15:20.018230+0000","last_fullsized":"2024-09-21T10:15:20.018230+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:33.267760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":133,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.039740+0000","last_change":"2024-09-21T10:13:56.570638+0000","last_active":"2024-09-21T10:15:21.039740+0000","last_peered":"2024-09-21T10:15:21.039740+0000","last_clean":"2024-09-21T10:15:21.039740+0000","last_became_active":"2024-09-21T10:13:56.570132+0000","last_became_peered":"2024-09-21T10:13:56.570132+0000","last_unstale":"2024-09-21T10:15:21.039740+0000","last_undegraded":"2024-09-21T10:15:21.039740+0000","last_fullsized":"2024-09-21T10:15:21.039740+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:28.451531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":107,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.032158+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:15:21.032158+0000","last_peered":"2024-09-21T10:15:21.032158+0000","last_clean":"2024-09-21T10:15:21.032158+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:15:21.032158+0000","last_undegraded":"2024-09-21T10:15:21.032158+0000","last_fullsized":"2024-09-21T10:15:21.032158+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"263'5115","reported_seq":5142,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.591520+0000","last_change":"2024-09-21T10:14:28.437169+0000","last_active":"2024-09-21T10:15:34.591520+0000","last_peered":"2024-09-21T10:15:34.591520+0000","last_clean":"2024-09-21T10:15:34.591520+0000","last_became_active":"2024-09-21T10:14:28.436873+0000","last_became_peered":"2024-09-21T10:14:28.436873+0000","last_unstale":"2024-09-21T10:15:34.591520+0000","last_undegraded":"2024-09-21T10:15:34.591520+0000","last_fullsized":"2024-09-21T10:15:34.591520+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5115,"log_dups_size":0,"ondisk_log_size":5115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:16.516429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335216640,"num_objects":5115,"num_object_clones":0,"num_object_copies":15345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5115,"num_write_kb":327360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":247,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.029388+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:15:21.029388+0000","last_peered":"2024-09-21T10:15:21.029388+0000","last_clean":"2024-09-21T10:15:21.029388+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:15:21.029388+0000","last_undegraded":"2024-09-21T10:15:21.029388+0000","last_fullsized":"2024-09-21T10:15:21.029388+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":684,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.032277+0000","last_change":"2024-09-21T10:13:59.577297+0000","last_active":"2024-09-21T10:15:21.032277+0000","last_peered":"2024-09-21T10:15:21.032277+0000","last_clean":"2024-09-21T10:15:21.032277+0000","last_became_active":"2024-09-21T10:13:59.576704+0000","last_became_peered":"2024-09-21T10:13:59.576704+0000","last_unstale":"2024-09-21T10:15:21.032277+0000","last_undegraded":"2024-09-21T10:15:21.032277+0000","last_fullsized":"2024-09-21T10:15:21.032277+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:11:28.773044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":121,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.033289+0000","last_change":"2024-09-21T10:14:45.531907+0000","last_active":"2024-09-21T10:15:21.033289+0000","last_peered":"2024-09-21T10:15:21.033289+0000","last_clean":"2024-09-21T10:15:21.033289+0000","last_became_active":"2024-09-21T10:14:45.531536+0000","last_became_peered":"2024-09-21T10:14:45.531536+0000","last_unstale":"2024-09-21T10:15:21.033289+0000","last_undegraded":"2024-09-21T10:15:21.033289+0000","last_fullsized":"2024-09-21T10:15:21.033289+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:18:52.155338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"263'5248","reported_seq":5577,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:34.691607+0000","last_change":"2024-09-21T10:14:48.563829+0000","last_active":"2024-09-21T10:15:34.691607+0000","last_peered":"2024-09-21T10:15:34.691607+0000","last_clean":"2024-09-21T10:15:34.691607+0000","last_became_active":"2024-09-21T10:14:48.549468+0000","last_became_peered":"2024-09-21T10:14:48.549468+0000","last_unstale":"2024-09-21T10:15:34.691607+0000","last_undegraded":"2024-09-21T10:15:34.691607+0000","last_fullsized":"2024-09-21T10:15:34.691607+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5248,"log_dups_size":0,"ondisk_log_size":5248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:08.229536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343932928,"num_objects":5248,"num_object_clones":0,"num_object_copies":15744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5248,"num_write_kb":335872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":114,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.032257+0000","last_change":"2024-09-21T10:14:45.530390+0000","last_active":"2024-09-21T10:15:21.032257+0000","last_peered":"2024-09-21T10:15:21.032257+0000","last_clean":"2024-09-21T10:15:21.032257+0000","last_became_active":"2024-09-21T10:14:45.530054+0000","last_became_peered":"2024-09-21T10:14:45.530054+0000","last_unstale":"2024-09-21T10:15:21.032257+0000","last_undegraded":"2024-09-21T10:15:21.032257+0000","last_fullsized":"2024-09-21T10:15:21.032257+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:42:49.775110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":744,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.014880+0000","last_change":"2024-09-21T10:13:09.604050+0000","last_active":"2024-09-21T10:15:20.014880+0000","last_peered":"2024-09-21T10:15:20.014880+0000","last_clean":"2024-09-21T10:15:20.014880+0000","last_became_active":"2024-09-21T10:13:09.583593+0000","last_became_peered":"2024-09-21T10:13:09.583593+0000","last_unstale":"2024-09-21T10:15:20.014880+0000","last_undegraded":"2024-09-21T10:15:20.014880+0000","last_fullsized":"2024-09-21T10:15:20.014880+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:35.420105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":327,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.018010+0000","last_change":"2024-09-21T10:14:45.528988+0000","last_active":"2024-09-21T10:15:20.018010+0000","last_peered":"2024-09-21T10:15:20.018010+0000","last_clean":"2024-09-21T10:15:20.018010+0000","last_became_active":"2024-09-21T10:14:45.528689+0000","last_became_peered":"2024-09-21T10:14:45.528689+0000","last_unstale":"2024-09-21T10:15:20.018010+0000","last_undegraded":"2024-09-21T10:15:20.018010+0000","last_fullsized":"2024-09-21T10:15:20.018010+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:48:43.539523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"263'5097","reported_seq":5144,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:32.577822+0000","last_change":"2024-09-21T10:14:28.438913+0000","last_active":"2024-09-21T10:15:32.577822+0000","last_peered":"2024-09-21T10:15:32.577822+0000","last_clean":"2024-09-21T10:15:32.577822+0000","last_became_active":"2024-09-21T10:14:28.438598+0000","last_became_peered":"2024-09-21T10:14:28.438598+0000","last_unstale":"2024-09-21T10:15:32.577822+0000","last_undegraded":"2024-09-21T10:15:32.577822+0000","last_fullsized":"2024-09-21T10:15:32.577822+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5097,"log_dups_size":0,"ondisk_log_size":5097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:29:24.765631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334036992,"num_objects":5097,"num_object_clones":0,"num_object_copies":15291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5097,"num_write_kb":326208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":395,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.045518+0000","last_change":"2024-09-21T10:13:56.647321+0000","last_active":"2024-09-21T10:15:21.045518+0000","last_peered":"2024-09-21T10:15:21.045518+0000","last_clean":"2024-09-21T10:15:21.045518+0000","last_became_active":"2024-09-21T10:13:56.644948+0000","last_became_peered":"2024-09-21T10:13:56.644948+0000","last_unstale":"2024-09-21T10:15:21.045518+0000","last_undegraded":"2024-09-21T10:15:21.045518+0000","last_fullsized":"2024-09-21T10:15:21.045518+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:40.992911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":711,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.032715+0000","last_change":"2024-09-21T10:13:12.569309+0000","last_active":"2024-09-21T10:15:21.032715+0000","last_peered":"2024-09-21T10:15:21.032715+0000","last_clean":"2024-09-21T10:15:21.032715+0000","last_became_active":"2024-09-21T10:13:12.568325+0000","last_became_peered":"2024-09-21T10:13:12.568325+0000","last_unstale":"2024-09-21T10:15:21.032715+0000","last_undegraded":"2024-09-21T10:15:21.032715+0000","last_fullsized":"2024-09-21T10:15:21.032715+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:18.314566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":402,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.030457+0000","last_change":"2024-09-21T10:13:09.579331+0000","last_active":"2024-09-21T10:15:21.030457+0000","last_peered":"2024-09-21T10:15:21.030457+0000","last_clean":"2024-09-21T10:15:21.030457+0000","last_became_active":"2024-09-21T10:13:09.578777+0000","last_became_peered":"2024-09-21T10:13:09.578777+0000","last_unstale":"2024-09-21T10:15:21.030457+0000","last_undegraded":"2024-09-21T10:15:21.030457+0000","last_fullsized":"2024-09-21T10:15:21.030457+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:50:34.425710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"263'5001","reported_seq":5048,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:32.566731+0000","last_change":"2024-09-21T10:14:28.439188+0000","last_active":"2024-09-21T10:15:32.566731+0000","last_peered":"2024-09-21T10:15:32.566731+0000","last_clean":"2024-09-21T10:15:32.566731+0000","last_became_active":"2024-09-21T10:14:28.438763+0000","last_became_peered":"2024-09-21T10:14:28.438763+0000","last_unstale":"2024-09-21T10:15:32.566731+0000","last_undegraded":"2024-09-21T10:15:32.566731+0000","last_fullsized":"2024-09-21T10:15:32.566731+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5001,"log_dups_size":0,"ondisk_log_size":5001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:31.383586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327745536,"num_objects":5001,"num_object_clones":0,"num_object_copies":15003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5001,"num_write_kb":320064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":111,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.032328+0000","last_change":"2024-09-21T10:13:09.592519+0000","last_active":"2024-09-21T10:15:21.032328+0000","last_peered":"2024-09-21T10:15:21.032328+0000","last_clean":"2024-09-21T10:15:21.032328+0000","last_became_active":"2024-09-21T10:13:09.587615+0000","last_became_peered":"2024-09-21T10:13:09.587615+0000","last_unstale":"2024-09-21T10:15:21.032328+0000","last_undegraded":"2024-09-21T10:15:21.032328+0000","last_fullsized":"2024-09-21T10:15:21.032328+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:22:43.216612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":507,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.015038+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:15:20.015038+0000","last_peered":"2024-09-21T10:15:20.015038+0000","last_clean":"2024-09-21T10:15:20.015038+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:15:20.015038+0000","last_undegraded":"2024-09-21T10:15:20.015038+0000","last_fullsized":"2024-09-21T10:15:20.015038+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":216,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.017198+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:15:20.017198+0000","last_peered":"2024-09-21T10:15:20.017198+0000","last_clean":"2024-09-21T10:15:20.017198+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:15:20.017198+0000","last_undegraded":"2024-09-21T10:15:20.017198+0000","last_fullsized":"2024-09-21T10:15:20.017198+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":256,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.038463+0000","last_change":"2024-09-21T10:15:21.038463+0000","last_active":"2024-09-21T10:15:21.038463+0000","last_peered":"2024-09-21T10:15:21.038463+0000","last_clean":"2024-09-21T10:15:21.038463+0000","last_became_active":"2024-09-21T10:15:21.038061+0000","last_became_peered":"2024-09-21T10:15:21.038061+0000","last_unstale":"2024-09-21T10:15:21.038463+0000","last_undegraded":"2024-09-21T10:15:21.038463+0000","last_fullsized":"2024-09-21T10:15:21.038463+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.241295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":238,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.017122+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:15:20.017122+0000","last_peered":"2024-09-21T10:15:20.017122+0000","last_clean":"2024-09-21T10:15:20.017122+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:15:20.017122+0000","last_undegraded":"2024-09-21T10:15:20.017122+0000","last_fullsized":"2024-09-21T10:15:20.017122+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.039692+0000","last_change":"2024-09-21T10:15:21.039385+0000","last_active":"2024-09-21T10:15:21.039692+0000","last_peered":"2024-09-21T10:15:21.039692+0000","last_clean":"2024-09-21T10:15:21.039692+0000","last_became_active":"2024-09-21T10:15:21.039073+0000","last_became_peered":"2024-09-21T10:15:21.039073+0000","last_unstale":"2024-09-21T10:15:21.039692+0000","last_undegraded":"2024-09-21T10:15:21.039692+0000","last_fullsized":"2024-09-21T10:15:21.039692+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:11.160836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":233,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.014651+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:15:20.014651+0000","last_peered":"2024-09-21T10:15:20.014651+0000","last_clean":"2024-09-21T10:15:20.014651+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:15:20.014651+0000","last_undegraded":"2024-09-21T10:15:20.014651+0000","last_fullsized":"2024-09-21T10:15:20.014651+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.018859+0000","last_change":"2024-09-21T10:14:45.528895+0000","last_active":"2024-09-21T10:15:20.018859+0000","last_peered":"2024-09-21T10:15:20.018859+0000","last_clean":"2024-09-21T10:15:20.018859+0000","last_became_active":"2024-09-21T10:14:45.528608+0000","last_became_peered":"2024-09-21T10:14:45.528608+0000","last_unstale":"2024-09-21T10:15:20.018859+0000","last_undegraded":"2024-09-21T10:15:20.018859+0000","last_fullsized":"2024-09-21T10:15:20.018859+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:48.887817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":334,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.030033+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:15:21.030033+0000","last_peered":"2024-09-21T10:15:21.030033+0000","last_clean":"2024-09-21T10:15:21.030033+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:15:21.030033+0000","last_undegraded":"2024-09-21T10:15:21.030033+0000","last_fullsized":"2024-09-21T10:15:21.030033+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.018804+0000","last_change":"2024-09-21T10:14:45.530392+0000","last_active":"2024-09-21T10:15:20.018804+0000","last_peered":"2024-09-21T10:15:20.018804+0000","last_clean":"2024-09-21T10:15:20.018804+0000","last_became_active":"2024-09-21T10:14:45.530079+0000","last_became_peered":"2024-09-21T10:14:45.530079+0000","last_unstale":"2024-09-21T10:15:20.018804+0000","last_undegraded":"2024-09-21T10:15:20.018804+0000","last_fullsized":"2024-09-21T10:15:20.018804+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:17.306916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":413,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.030410+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:15:21.030410+0000","last_peered":"2024-09-21T10:15:21.030410+0000","last_clean":"2024-09-21T10:15:21.030410+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:15:21.030410+0000","last_undegraded":"2024-09-21T10:15:21.030410+0000","last_fullsized":"2024-09-21T10:15:21.030410+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":258,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.016132+0000","last_change":"2024-09-21T10:13:09.573620+0000","last_active":"2024-09-21T10:15:20.016132+0000","last_peered":"2024-09-21T10:15:20.016132+0000","last_clean":"2024-09-21T10:15:20.016132+0000","last_became_active":"2024-09-21T10:13:09.573216+0000","last_became_peered":"2024-09-21T10:13:09.573216+0000","last_unstale":"2024-09-21T10:15:20.016132+0000","last_undegraded":"2024-09-21T10:15:20.016132+0000","last_fullsized":"2024-09-21T10:15:20.016132+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:40.253863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":422,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.030535+0000","last_change":"2024-09-21T10:13:09.581932+0000","last_active":"2024-09-21T10:15:21.030535+0000","last_peered":"2024-09-21T10:15:21.030535+0000","last_clean":"2024-09-21T10:15:21.030535+0000","last_became_active":"2024-09-21T10:13:09.580798+0000","last_became_peered":"2024-09-21T10:13:09.580798+0000","last_unstale":"2024-09-21T10:15:21.030535+0000","last_undegraded":"2024-09-21T10:15:21.030535+0000","last_fullsized":"2024-09-21T10:15:21.030535+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:05.563826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":107,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.033453+0000","last_change":"2024-09-21T10:15:21.033453+0000","last_active":"2024-09-21T10:15:21.033453+0000","last_peered":"2024-09-21T10:15:21.033453+0000","last_clean":"2024-09-21T10:15:21.033453+0000","last_became_active":"2024-09-21T10:15:21.032893+0000","last_became_peered":"2024-09-21T10:15:21.032893+0000","last_unstale":"2024-09-21T10:15:21.033453+0000","last_undegraded":"2024-09-21T10:15:21.033453+0000","last_fullsized":"2024-09-21T10:15:21.033453+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:14.164715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":114,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.032788+0000","last_change":"2024-09-21T10:14:32.454735+0000","last_active":"2024-09-21T10:15:21.032788+0000","last_peered":"2024-09-21T10:15:21.032788+0000","last_clean":"2024-09-21T10:15:21.032788+0000","last_became_active":"2024-09-21T10:14:32.454475+0000","last_became_peered":"2024-09-21T10:14:32.454475+0000","last_unstale":"2024-09-21T10:15:21.032788+0000","last_undegraded":"2024-09-21T10:15:21.032788+0000","last_fullsized":"2024-09-21T10:15:21.032788+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:59.191649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":405,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.035195+0000","last_change":"2024-09-21T10:14:20.761163+0000","last_active":"2024-09-21T10:15:21.035195+0000","last_peered":"2024-09-21T10:15:21.035195+0000","last_clean":"2024-09-21T10:15:21.035195+0000","last_became_active":"2024-09-21T10:14:20.760814+0000","last_became_peered":"2024-09-21T10:14:20.760814+0000","last_unstale":"2024-09-21T10:15:21.035195+0000","last_undegraded":"2024-09-21T10:15:21.035195+0000","last_fullsized":"2024-09-21T10:15:21.035195+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:44.679743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":264,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.016983+0000","last_change":"2024-09-21T10:13:09.577994+0000","last_active":"2024-09-21T10:15:20.016983+0000","last_peered":"2024-09-21T10:15:20.016983+0000","last_clean":"2024-09-21T10:15:20.016983+0000","last_became_active":"2024-09-21T10:13:09.577311+0000","last_became_peered":"2024-09-21T10:13:09.577311+0000","last_unstale":"2024-09-21T10:15:20.016983+0000","last_undegraded":"2024-09-21T10:15:20.016983+0000","last_fullsized":"2024-09-21T10:15:20.016983+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:27.532202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":418,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.037690+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:15:21.037690+0000","last_peered":"2024-09-21T10:15:21.037690+0000","last_clean":"2024-09-21T10:15:21.037690+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:15:21.037690+0000","last_undegraded":"2024-09-21T10:15:21.037690+0000","last_fullsized":"2024-09-21T10:15:21.037690+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":143,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.017528+0000","last_change":"2024-09-21T10:13:09.578314+0000","last_active":"2024-09-21T10:15:20.017528+0000","last_peered":"2024-09-21T10:15:20.017528+0000","last_clean":"2024-09-21T10:15:20.017528+0000","last_became_active":"2024-09-21T10:13:09.577570+0000","last_became_peered":"2024-09-21T10:13:09.577570+0000","last_unstale":"2024-09-21T10:15:20.017528+0000","last_undegraded":"2024-09-21T10:15:20.017528+0000","last_fullsized":"2024-09-21T10:15:20.017528+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:21.299713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":109,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.017326+0000","last_change":"2024-09-21T10:13:09.577841+0000","last_active":"2024-09-21T10:15:20.017326+0000","last_peered":"2024-09-21T10:15:20.017326+0000","last_clean":"2024-09-21T10:15:20.017326+0000","last_became_active":"2024-09-21T10:13:09.576793+0000","last_became_peered":"2024-09-21T10:13:09.576793+0000","last_unstale":"2024-09-21T10:15:20.017326+0000","last_undegraded":"2024-09-21T10:15:20.017326+0000","last_fullsized":"2024-09-21T10:15:20.017326+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:12.718520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":258,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.029432+0000","last_change":"2024-09-21T10:13:09.589654+0000","last_active":"2024-09-21T10:15:21.029432+0000","last_peered":"2024-09-21T10:15:21.029432+0000","last_clean":"2024-09-21T10:15:21.029432+0000","last_became_active":"2024-09-21T10:13:09.574438+0000","last_became_peered":"2024-09-21T10:13:09.574438+0000","last_unstale":"2024-09-21T10:15:21.029432+0000","last_undegraded":"2024-09-21T10:15:21.029432+0000","last_fullsized":"2024-09-21T10:15:21.029432+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:12.504161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.016762+0000","last_change":"2024-09-21T10:14:32.771090+0000","last_active":"2024-09-21T10:15:20.016762+0000","last_peered":"2024-09-21T10:15:20.016762+0000","last_clean":"2024-09-21T10:15:20.016762+0000","last_became_active":"2024-09-21T10:14:32.770619+0000","last_became_peered":"2024-09-21T10:14:32.770619+0000","last_unstale":"2024-09-21T10:15:20.016762+0000","last_undegraded":"2024-09-21T10:15:20.016762+0000","last_fullsized":"2024-09-21T10:15:20.016762+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:29:31.473095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":217,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.029296+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:15:21.029296+0000","last_peered":"2024-09-21T10:15:21.029296+0000","last_clean":"2024-09-21T10:15:21.029296+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:15:21.029296+0000","last_undegraded":"2024-09-21T10:15:21.029296+0000","last_fullsized":"2024-09-21T10:15:21.029296+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":425,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.030288+0000","last_change":"2024-09-21T10:14:45.526997+0000","last_active":"2024-09-21T10:15:21.030288+0000","last_peered":"2024-09-21T10:15:21.030288+0000","last_clean":"2024-09-21T10:15:21.030288+0000","last_became_active":"2024-09-21T10:14:45.526600+0000","last_became_peered":"2024-09-21T10:14:45.526600+0000","last_unstale":"2024-09-21T10:15:21.030288+0000","last_undegraded":"2024-09-21T10:15:21.030288+0000","last_fullsized":"2024-09-21T10:15:21.030288+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:33:20.585859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":393,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-21T10:15:21.036227+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:15:21.036227+0000","last_peered":"2024-09-21T10:15:21.036227+0000","last_clean":"2024-09-21T10:15:21.036227+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:15:21.036227+0000","last_undegraded":"2024-09-21T10:15:21.036227+0000","last_fullsized":"2024-09-21T10:15:21.036227+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":247,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.016769+0000","last_change":"2024-09-21T10:13:09.577388+0000","last_active":"2024-09-21T10:15:20.016769+0000","last_peered":"2024-09-21T10:15:20.016769+0000","last_clean":"2024-09-21T10:15:20.016769+0000","last_became_active":"2024-09-21T10:13:09.577027+0000","last_became_peered":"2024-09-21T10:13:09.577027+0000","last_unstale":"2024-09-21T10:15:20.016769+0000","last_undegraded":"2024-09-21T10:15:20.016769+0000","last_fullsized":"2024-09-21T10:15:20.016769+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:49.522609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":27,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-09-21T10:15:20.018803+0000","last_change":"2024-09-21T10:14:45.536046+0000","last_active":"2024-09-21T10:15:20.018803+0000","last_peered":"2024-09-21T10:15:20.018803+0000","last_clean":"2024-09-21T10:15:20.018803+0000","last_became_active":"2024-09-21T10:14:45.530295+0000","last_became_peered":"2024-09-21T10:14:45.530295+0000","last_unstale":"2024-09-21T10:15:20.018803+0000","last_undegraded":"2024-09-21T10:15:20.018803+0000","last_fullsized":"2024-09-21T10:15:20.018803+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:34.920194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5395775488,"num_objects":82333,"num_object_clones":0,"num_object_copies":246999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82333,"num_write_kb":5269312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16205873152,"data_stored":16205873152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82333,"ondisk_log_size":82333,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":1370,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":12582950,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234311,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2400144,"kb_used_data":1985928,"kb_used_omap":1,"kb_used_meta":414206,"kb_avail":91345008,"statfs":{"total":95995035648,"available":93537288192,"internally_reserved":0,"allocated":2033590272,"data_stored":2032348569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":424147403},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495949,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2341776,"kb_used_data":1963656,"kb_used_omap":1,"kb_used_meta":378110,"kb_avail":91403376,"statfs":{"total":95995035648,"available":93597057024,"internally_reserved":0,"allocated":2010783744,"data_stored":2009542827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":387185095},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757586,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1994112,"kb_used_data":1647992,"kb_used_omap":1,"kb_used_meta":346110,"kb_avail":91751040,"statfs":{"total":95995035648,"available":93953064960,"internally_reserved":0,"allocated":1687543808,"data_stored":1686313975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":354417094},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019222,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2355340,"kb_used_data":1999364,"kb_used_omap":1,"kb_used_meta":355966,"kb_avail":91389812,"statfs":{"total":95995035648,"available":93583167488,"internally_reserved":0,"allocated":2047348736,"data_stored":2046110846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":364509638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435406,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3979192,"kb_used_data":3506032,"kb_used_omap":5,"kb_used_meta":473146,"kb_avail":89765960,"statfs":{"total":95995035648,"available":91920343040,"internally_reserved":0,"allocated":3590176768,"data_stored":3588954558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":484501544},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598933,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2421240,"kb_used_data":1970288,"kb_used_omap":7,"kb_used_meta":450936,"kb_avail":91323912,"statfs":{"total":95995035648,"available":93515685888,"internally_reserved":0,"allocated":2017574912,"data_stored":2016352920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":461758925},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762460,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2967292,"kb_used_data":2650996,"kb_used_omap":7,"kb_used_meta":316280,"kb_avail":90777860,"statfs":{"total":95995035648,"available":92956528640,"internally_reserved":0,"allocated":2714619904,"data_stored":2713393820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":323870973},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":403726925986,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2729836544,"data_stored":2729836544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2019295232,"data_stored":2019295232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3624665088,"data_stored":3624665088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2048065536,"data_stored":2048065536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1707409408,"data_stored":1707409408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2029191168,"data_stored":2029191168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2047410176,"data_stored":2047410176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:15:39.687 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:15:39.688 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:39.719 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:39.820 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:39.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 56 16 89425 89409 99.7712 74.375 0.0243851 0.0100176 2024-09-21T10:15:39.866 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:15:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:15:39] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.51.0" 2024-09-21T10:15:39.968 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:40.004 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:40.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:40.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:40.144 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:40.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:40.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:40.287 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:40.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:40.388 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:40.422 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:40.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:40.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:40 smithi032 ceph-mon[105748]: pgmap v757: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:15:40.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:40 smithi032 ceph-mon[101299]: pgmap v757: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:15:40.525 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:40.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:40 smithi045 ceph-mon[95976]: pgmap v757: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:15:40.563 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:40.665 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:40.700 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:40.802 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:40.839 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:40.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 57 16 90624 90608 99.3354 74.9375 0.00851442 0.0100625 2024-09-21T10:15:40.942 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:40.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:41.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:41.108 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:41.124 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:41.210 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:41.243 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:41.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:41.282 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:41 smithi032 ceph-mon[101299]: pgmap v758: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:15:41.282 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:41 smithi032 ceph-mon[105748]: pgmap v758: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:15:41.344 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:41.380 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:41.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:41.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:41.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:41.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:41.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:41 smithi045 ceph-mon[95976]: pgmap v758: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:15:41.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:41.648 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:41.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:41.750 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:41.789 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:41.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:41.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 58 16 91737 91721 98.822 69.5625 0.00450839 0.0101138 2024-09-21T10:15:41.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:41.923 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:41.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:42.004 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:15:42.004 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:42.025 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:42.041 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:15:42.041 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:42.056 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:42.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:42.074 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:15:42.074 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:42.107 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:15:42.107 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:15:42.142 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:15:42.143 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:42.158 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:42.178 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:15:42.178 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:42.188 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:42.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:42.217 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:15:42.217 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:42.262 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:15:42.262 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:42.290 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:42.333 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:42.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:42.345 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:42 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3969877286' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:42.345 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:42 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3969877286' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:42.435 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:42.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:42.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:42.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:42 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3969877286' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:42.578 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:42.620 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:42.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:42.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:42.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:42.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:42.855 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:42.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 59 16 92864 92848 98.3408 70.4375 0.0163761 0.0101639 2024-09-21T10:15:42.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:42.991 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:43.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:43.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:43.116 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:43.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:43.151 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:43.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:43.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:43.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:43.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:43.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:43 smithi032 ceph-mon[105748]: pgmap v759: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:15:43.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:43 smithi032 ceph-mon[101299]: pgmap v759: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:15:43.470 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:43 smithi045 ceph-mon[95976]: pgmap v759: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:15:43.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:43.611 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:43.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:43.713 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:43.755 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:43.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:43.856 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:43.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:15:43.856266+0000 min lat: 0.00117819 max lat: 2.28911 avg lat: 0.0102053 2024-09-21T10:15:43.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:15:43.858 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 60 16 94051 94035 97.9382 74.1875 0.00750234 0.0102053 2024-09-21T10:15:43.891 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:43.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:43.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:44.029 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:44.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:44.173 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:44.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:44.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:44.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:44.412 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:44.440 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:44.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:44.542 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:44.575 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:44.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:44.677 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:44.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:44.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:44.813 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:44.844 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:44.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 61 16 95220 95204 97.5303 73.0625 0.0152859 0.0102482 2024-09-21T10:15:44.946 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:44.982 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:44.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:45.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:45.127 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:45.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:45.229 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:45.267 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:45.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:45.369 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:45.417 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:45.519 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:45.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:45.658 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:45.695 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:45.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:45.696 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:45 smithi032 ceph-mon[105748]: pgmap v760: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:15:45.696 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:45 smithi032 ceph-mon[101299]: pgmap v760: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:15:45.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:45 smithi045 ceph-mon[95976]: pgmap v760: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-21T10:15:45.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.838 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:45.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:45.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 62 16 96425 96409 97.1719 75.3125 0.0126786 0.0102862 2024-09-21T10:15:45.927 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:15:45.939 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:45.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:46.071 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.139 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:46.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:46.240 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:46.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:46.387 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:46.421 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:46.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:46.523 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:46.560 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:46.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:46.662 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:46.691 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:46.794 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:46.826 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:46.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:46.858 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 63 16 97536 97520 96.7316 69.4375 0.0050007 0.0103325 2024-09-21T10:15:46.927 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:46.966 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:46.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:47.068 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:47.106 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:47.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:47.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:47.245 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:47.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:47.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:47.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.414 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:47.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:47.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:47.548 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:47.650 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:47.656 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:15:47.656 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:15:47.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.656 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:98406 7.2 e264/245 7:486ff5d9:::benchmark_data_smithi032_200324_object98405:head v 264'6115, mlcod=264'6113)", 2024-09-21T10:15:47.656 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:15:47.649926+0000", 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.001508421, 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0015365800000000001, 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.649926+0000", 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:15:47.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.649926+0000", 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.649927+0000", 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.37e-06 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.649958+0000", 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0136000000000001e-05 2024-09-21T10:15:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.649980+0000", 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2138e-05 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.649989+0000", 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4569999999999994e-06 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650298+0000", 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00030881399999999997 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650320+0000", 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1868000000000001e-05 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:98409 7.8 7:183f14ea:::benchmark_data_smithi032_200324_object98408:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:15:47.650020+0000", 2024-09-21T10:15:47.660 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0014145550000000001, 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001503246, 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 98409 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650020+0000", 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650020+0000", 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650021+0000", 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5239999999999999e-06 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650075+0000", 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3875999999999997e-05 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650086+0000", 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0827e-05 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650094+0000", 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9820000000000008e-06 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.651387+0000", 2024-09-21T10:15:47.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001292755 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.651530+0000", 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000143232 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:98410 7.5 e264/245 7:aec75993:::benchmark_data_smithi032_200324_object98409:head v 264'6105, mlcod=264'6103)", 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:15:47.650120+0000", 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0013149450000000001, 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014607800000000001, 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:15:47.664 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650120+0000", 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650120+0000", 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650120+0000", 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5200000000000002e-07 2024-09-21T10:15:47.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650151+0000", 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1435e-05 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650159+0000", 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.272e-06 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650164+0000", 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9509999999999997e-06 2024-09-21T10:15:47.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650417+0000", 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00025301399999999997 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:15:47.650441+0000", 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4842999999999999e-05 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 3 2024-09-21T10:15:47.667 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:15:47.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.693 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:47.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:47.794 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:47.830 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:47.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:47.857 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 64 16 98633 98617 96.2914 68.5625 0.00783591 0.0103801 2024-09-21T10:15:47.892 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:47 smithi032 ceph-mon[101299]: pgmap v761: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-21T10:15:47.892 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:47 smithi032 ceph-mon[105748]: pgmap v761: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-21T10:15:47.932 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:47.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.975 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:47.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:47 smithi045 ceph-mon[95976]: pgmap v761: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-21T10:15:48.076 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:48.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:48.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:48.212 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:48.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.253 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:48.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:48.354 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:48.387 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:48.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:48.489 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:48.517 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:48.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:48.619 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:48.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.665 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:48.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:48.766 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:48.791 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:48.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:48.859 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 65 16 99597 99581 95.7368 60.25 0.0166915 0.0104398 2024-09-21T10:15:48.893 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:48.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.940 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:48.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:49.040 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:49.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:49.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:49.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:49.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:49.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:49.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:49.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:49.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:49.496 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:49.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:49.597 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:49.625 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:49.727 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:49.770 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:49 smithi045 ceph-mon[95976]: pgmap v762: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:15:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:49 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/399210400' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:49 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:49.858 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 66 16 100500 100484 95.1413 56.4375 0.0119553 0.0105052 2024-09-21T10:15:49.872 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:49.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.910 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:49.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:15:49] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-09-21T10:15:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[105748]: pgmap v762: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:15:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/399210400' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[101299]: pgmap v762: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:15:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:15:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/399210400' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:49 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:50.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:50.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.051 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:50.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:50.152 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:50.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.193 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:50.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:50.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:50.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:50.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:50.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:50.462 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:50.563 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:50.600 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:50.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:50 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:50.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:50 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:15:50.680 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:50 smithi032 ceph-mon[101299]: osdmap e265: 8 total, 8 up, 7 in 2024-09-21T10:15:50.681 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:50 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:50.681 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:50 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:15:50.681 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:50 smithi032 ceph-mon[105748]: osdmap e265: 8 total, 8 up, 7 in 2024-09-21T10:15:50.681 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:15:50.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:50.743 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:50.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:50.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:50 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:50.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:50 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:15:50.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:50 smithi045 ceph-mon[95976]: osdmap e265: 8 total, 8 up, 7 in 2024-09-21T10:15:50.845 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:50.858 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 67 16 101476 101460 94.6317 61 0.0149212 0.0105622 2024-09-21T10:15:50.876 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:50.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:50.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:51.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:51.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:51.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:51.166 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:51.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:51.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:51.308 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:51.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:51.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:51.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:51.467 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:15:51.547 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:51.580 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:51.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:51.681 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:51.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:51 smithi032 ceph-mon[105748]: pgmap v764: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:15:51.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:51 smithi032 ceph-mon[101299]: pgmap v764: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:15:51.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:51.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:51.858 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 68 16 102519 102503 94.1986 65.1875 0.00818401 0.0106102 2024-09-21T10:15:51.872 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:51.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:51.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:52.005 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:52.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:51 smithi045 ceph-mon[95976]: pgmap v764: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:15:52.102 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:15:52.103 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:15:52.107 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:52.155 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:52.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:52.258 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:52.292 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:52.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:52.394 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:52.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:52.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:52.570 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:52.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:52.672 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:52.711 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:52.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:52.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:52.847 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:52.858 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 69 16 103618 103602 93.8288 68.6875 0.00806127 0.0106523 2024-09-21T10:15:52.950 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:52.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:52 smithi032 ceph-mon[101299]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:52.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:52 smithi032 ceph-mon[105748]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:52.991 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:52.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:53.022 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":764,"stamp":"2024-09-21T10:15:50.374998+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6544562149,"num_objects":99817,"num_object_clones":0,"num_object_copies":299451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":99817,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":100912,"num_write_kb":6401537,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":927,"num_bytes_recovered":65475136,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100771,"ondisk_log_size":100771,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":22010476,"kb_used_data":19090932,"kb_used_omap":27,"kb_used_meta":2919460,"kb_avail":634205588,"statfs":{"total":671965249536,"available":649426522112,"internally_reserved":0,"allocated":19549114368,"data_stored":19540466131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28144,"internal_metadata":2989527568},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,6,4,14,8,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":744095744,"num_objects":11354,"num_object_clones":0,"num_object_copies":34062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11354,"num_write_kb":726656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002584"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":402,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112704+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:15:36.112704+0000","last_peered":"2024-09-21T10:15:36.112704+0000","last_clean":"2024-09-21T10:15:36.112704+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:15:36.112704+0000","last_undegraded":"2024-09-21T10:15:36.112704+0000","last_fullsized":"2024-09-21T10:15:36.112704+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":234,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115552+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:15:36.115552+0000","last_peered":"2024-09-21T10:15:36.115552+0000","last_clean":"2024-09-21T10:15:36.115552+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:15:36.115552+0000","last_undegraded":"2024-09-21T10:15:36.115552+0000","last_fullsized":"2024-09-21T10:15:36.115552+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":23,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112922+0000","last_change":"2024-09-21T10:14:45.529072+0000","last_active":"2024-09-21T10:15:36.112922+0000","last_peered":"2024-09-21T10:15:36.112922+0000","last_clean":"2024-09-21T10:15:36.112922+0000","last_became_active":"2024-09-21T10:14:45.528270+0000","last_became_peered":"2024-09-21T10:14:45.528270+0000","last_unstale":"2024-09-21T10:15:36.112922+0000","last_undegraded":"2024-09-21T10:15:36.112922+0000","last_fullsized":"2024-09-21T10:15:36.112922+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:00.326719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115256+0000","last_change":"2024-09-21T10:13:09.583319+0000","last_active":"2024-09-21T10:15:36.115256+0000","last_peered":"2024-09-21T10:15:36.115256+0000","last_clean":"2024-09-21T10:15:36.115256+0000","last_became_active":"2024-09-21T10:13:09.582953+0000","last_became_peered":"2024-09-21T10:13:09.582953+0000","last_unstale":"2024-09-21T10:15:36.115256+0000","last_undegraded":"2024-09-21T10:15:36.115256+0000","last_fullsized":"2024-09-21T10:15:36.115256+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:13.630031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":229,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.114965+0000","last_change":"2024-09-21T10:09:35.958207+0000","last_active":"2024-09-21T10:15:36.114965+0000","last_peered":"2024-09-21T10:15:36.114965+0000","last_clean":"2024-09-21T10:15:36.114965+0000","last_became_active":"2024-09-21T10:09:35.957860+0000","last_became_peered":"2024-09-21T10:09:35.957860+0000","last_unstale":"2024-09-21T10:15:36.114965+0000","last_undegraded":"2024-09-21T10:15:36.114965+0000","last_fullsized":"2024-09-21T10:15:36.114965+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:37.767930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":217,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118062+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:15:36.118062+0000","last_peered":"2024-09-21T10:15:36.118062+0000","last_clean":"2024-09-21T10:15:36.118062+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:15:36.118062+0000","last_undegraded":"2024-09-21T10:15:36.118062+0000","last_fullsized":"2024-09-21T10:15:36.118062+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":408,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.111719+0000","last_change":"2024-09-21T10:14:20.761360+0000","last_active":"2024-09-21T10:15:36.111719+0000","last_peered":"2024-09-21T10:15:36.111719+0000","last_clean":"2024-09-21T10:15:36.111719+0000","last_became_active":"2024-09-21T10:14:20.760941+0000","last_became_peered":"2024-09-21T10:14:20.760941+0000","last_unstale":"2024-09-21T10:15:36.111719+0000","last_undegraded":"2024-09-21T10:15:36.111719+0000","last_fullsized":"2024-09-21T10:15:36.111719+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:27.842256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":282,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115324+0000","last_change":"2024-09-21T10:14:45.537300+0000","last_active":"2024-09-21T10:15:36.115324+0000","last_peered":"2024-09-21T10:15:36.115324+0000","last_clean":"2024-09-21T10:15:36.115324+0000","last_became_active":"2024-09-21T10:14:45.527447+0000","last_became_peered":"2024-09-21T10:14:45.527447+0000","last_unstale":"2024-09-21T10:15:36.115324+0000","last_undegraded":"2024-09-21T10:15:36.115324+0000","last_fullsized":"2024-09-21T10:15:36.115324+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:33.652169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"265'6276","reported_seq":6615,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.837259+0000","last_change":"2024-09-21T10:14:49.554195+0000","last_active":"2024-09-21T10:15:49.837259+0000","last_peered":"2024-09-21T10:15:49.837259+0000","last_clean":"2024-09-21T10:15:49.837259+0000","last_became_active":"2024-09-21T10:14:49.551887+0000","last_became_peered":"2024-09-21T10:14:49.551887+0000","last_unstale":"2024-09-21T10:15:49.837259+0000","last_undegraded":"2024-09-21T10:15:49.837259+0000","last_fullsized":"2024-09-21T10:15:49.837259+0000","mapping_epoch":256,"log_start":"251'100","ondisk_log_start":"251'100","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6176,"log_dups_size":0,"ondisk_log_size":6176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:26.931226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411303936,"num_objects":6276,"num_object_clones":0,"num_object_copies":18828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6276,"num_write_kb":401664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":9043968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":273,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116402+0000","last_change":"2024-09-21T10:13:09.586457+0000","last_active":"2024-09-21T10:15:36.116402+0000","last_peered":"2024-09-21T10:15:36.116402+0000","last_clean":"2024-09-21T10:15:36.116402+0000","last_became_active":"2024-09-21T10:13:09.574624+0000","last_became_peered":"2024-09-21T10:13:09.574624+0000","last_unstale":"2024-09-21T10:15:36.116402+0000","last_undegraded":"2024-09-21T10:15:36.116402+0000","last_fullsized":"2024-09-21T10:15:36.116402+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:39:00.778086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":23,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112412+0000","last_change":"2024-09-21T10:14:45.530298+0000","last_active":"2024-09-21T10:15:36.112412+0000","last_peered":"2024-09-21T10:15:36.112412+0000","last_clean":"2024-09-21T10:15:36.112412+0000","last_became_active":"2024-09-21T10:14:45.529912+0000","last_became_peered":"2024-09-21T10:14:45.529912+0000","last_unstale":"2024-09-21T10:15:36.112412+0000","last_undegraded":"2024-09-21T10:15:36.112412+0000","last_fullsized":"2024-09-21T10:15:36.112412+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:50.567684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"264'6157","reported_seq":6526,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:47.615311+0000","last_change":"2024-09-21T10:14:49.553650+0000","last_active":"2024-09-21T10:15:47.615311+0000","last_peered":"2024-09-21T10:15:47.615311+0000","last_clean":"2024-09-21T10:15:47.615311+0000","last_became_active":"2024-09-21T10:14:49.552007+0000","last_became_peered":"2024-09-21T10:14:49.552007+0000","last_unstale":"2024-09-21T10:15:47.615311+0000","last_undegraded":"2024-09-21T10:15:47.615311+0000","last_fullsized":"2024-09-21T10:15:47.615311+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6157,"log_dups_size":0,"ondisk_log_size":6157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:43:27.336777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403505152,"num_objects":6157,"num_object_clones":0,"num_object_copies":18471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6157,"num_write_kb":394048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":300,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115506+0000","last_change":"2024-09-21T10:13:09.596445+0000","last_active":"2024-09-21T10:15:36.115506+0000","last_peered":"2024-09-21T10:15:36.115506+0000","last_clean":"2024-09-21T10:15:36.115506+0000","last_became_active":"2024-09-21T10:13:09.582665+0000","last_became_peered":"2024-09-21T10:13:09.582665+0000","last_unstale":"2024-09-21T10:15:36.115506+0000","last_undegraded":"2024-09-21T10:15:36.115506+0000","last_fullsized":"2024-09-21T10:15:36.115506+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:23:47.510351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":265,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118857+0000","last_change":"2024-09-21T10:13:09.578819+0000","last_active":"2024-09-21T10:15:36.118857+0000","last_peered":"2024-09-21T10:15:36.118857+0000","last_clean":"2024-09-21T10:15:36.118857+0000","last_became_active":"2024-09-21T10:13:09.576707+0000","last_became_peered":"2024-09-21T10:13:09.576707+0000","last_unstale":"2024-09-21T10:15:36.118857+0000","last_undegraded":"2024-09-21T10:15:36.118857+0000","last_fullsized":"2024-09-21T10:15:36.118857+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:19.175846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"265'6296","reported_seq":6354,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.786651+0000","last_change":"2024-09-21T10:14:32.771740+0000","last_active":"2024-09-21T10:15:49.786651+0000","last_peered":"2024-09-21T10:15:49.786651+0000","last_clean":"2024-09-21T10:15:49.786651+0000","last_became_active":"2024-09-21T10:14:32.771298+0000","last_became_peered":"2024-09-21T10:14:32.771298+0000","last_unstale":"2024-09-21T10:15:49.786651+0000","last_undegraded":"2024-09-21T10:15:49.786651+0000","last_fullsized":"2024-09-21T10:15:49.786651+0000","mapping_epoch":248,"log_start":"251'100","ondisk_log_start":"251'100","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6196,"log_dups_size":0,"ondisk_log_size":6196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:27.287728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412614656,"num_objects":6296,"num_object_clones":0,"num_object_copies":18888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6296,"num_write_kb":402944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":245,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116519+0000","last_change":"2024-09-21T10:14:45.533171+0000","last_active":"2024-09-21T10:15:36.116519+0000","last_peered":"2024-09-21T10:15:36.116519+0000","last_clean":"2024-09-21T10:15:36.116519+0000","last_became_active":"2024-09-21T10:14:45.527709+0000","last_became_peered":"2024-09-21T10:14:45.527709+0000","last_unstale":"2024-09-21T10:15:36.116519+0000","last_undegraded":"2024-09-21T10:15:36.116519+0000","last_fullsized":"2024-09-21T10:15:36.116519+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:33.989738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":250,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115248+0000","last_change":"2024-09-21T10:13:09.578358+0000","last_active":"2024-09-21T10:15:36.115248+0000","last_peered":"2024-09-21T10:15:36.115248+0000","last_clean":"2024-09-21T10:15:36.115248+0000","last_became_active":"2024-09-21T10:13:09.577273+0000","last_became_peered":"2024-09-21T10:13:09.577273+0000","last_unstale":"2024-09-21T10:15:36.115248+0000","last_undegraded":"2024-09-21T10:15:36.115248+0000","last_fullsized":"2024-09-21T10:15:36.115248+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:17.971932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"264'6235","reported_seq":6284,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:48.819018+0000","last_change":"2024-09-21T10:14:28.438058+0000","last_active":"2024-09-21T10:15:48.819018+0000","last_peered":"2024-09-21T10:15:48.819018+0000","last_clean":"2024-09-21T10:15:48.819018+0000","last_became_active":"2024-09-21T10:14:28.437772+0000","last_became_peered":"2024-09-21T10:14:28.437772+0000","last_unstale":"2024-09-21T10:15:48.819018+0000","last_undegraded":"2024-09-21T10:15:48.819018+0000","last_fullsized":"2024-09-21T10:15:48.819018+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6235,"log_dups_size":0,"ondisk_log_size":6235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:14.437061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408616960,"num_objects":6235,"num_object_clones":0,"num_object_copies":18705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6235,"num_write_kb":399040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":233,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112701+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:15:36.112701+0000","last_peered":"2024-09-21T10:15:36.112701+0000","last_clean":"2024-09-21T10:15:36.112701+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:15:36.112701+0000","last_undegraded":"2024-09-21T10:15:36.112701+0000","last_fullsized":"2024-09-21T10:15:36.112701+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":363,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.606024+0000","last_change":"2024-09-21T10:13:56.569595+0000","last_active":"2024-09-21T10:15:49.606024+0000","last_peered":"2024-09-21T10:15:49.606024+0000","last_clean":"2024-09-21T10:15:49.606024+0000","last_became_active":"2024-09-21T10:13:56.567665+0000","last_became_peered":"2024-09-21T10:13:56.567665+0000","last_unstale":"2024-09-21T10:15:49.606024+0000","last_undegraded":"2024-09-21T10:15:49.606024+0000","last_fullsized":"2024-09-21T10:15:49.606024+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:12:35.947882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"264'6172","reported_seq":6221,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:47.513712+0000","last_change":"2024-09-21T10:14:28.441282+0000","last_active":"2024-09-21T10:15:47.513712+0000","last_peered":"2024-09-21T10:15:47.513712+0000","last_clean":"2024-09-21T10:15:47.513712+0000","last_became_active":"2024-09-21T10:14:28.440767+0000","last_became_peered":"2024-09-21T10:14:28.440767+0000","last_unstale":"2024-09-21T10:15:47.513712+0000","last_undegraded":"2024-09-21T10:15:47.513712+0000","last_fullsized":"2024-09-21T10:15:47.513712+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6172,"log_dups_size":0,"ondisk_log_size":6172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:56:18.651559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404488192,"num_objects":6172,"num_object_clones":0,"num_object_copies":18516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6172,"num_write_kb":395008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":419,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112927+0000","last_change":"2024-09-21T10:13:09.594302+0000","last_active":"2024-09-21T10:15:36.112927+0000","last_peered":"2024-09-21T10:15:36.112927+0000","last_clean":"2024-09-21T10:15:36.112927+0000","last_became_active":"2024-09-21T10:13:09.576948+0000","last_became_peered":"2024-09-21T10:13:09.576948+0000","last_unstale":"2024-09-21T10:15:36.112927+0000","last_undegraded":"2024-09-21T10:15:36.112927+0000","last_fullsized":"2024-09-21T10:15:36.112927+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:10.360060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":232,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119317+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:15:36.119317+0000","last_peered":"2024-09-21T10:15:36.119317+0000","last_clean":"2024-09-21T10:15:36.119317+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:15:36.119317+0000","last_undegraded":"2024-09-21T10:15:36.119317+0000","last_fullsized":"2024-09-21T10:15:36.119317+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"264'6283","reported_seq":6677,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:48.817951+0000","last_change":"2024-09-21T10:14:49.553783+0000","last_active":"2024-09-21T10:15:48.817951+0000","last_peered":"2024-09-21T10:15:48.817951+0000","last_clean":"2024-09-21T10:15:48.817951+0000","last_became_active":"2024-09-21T10:14:49.552803+0000","last_became_peered":"2024-09-21T10:14:49.552803+0000","last_unstale":"2024-09-21T10:15:48.817951+0000","last_undegraded":"2024-09-21T10:15:48.817951+0000","last_fullsized":"2024-09-21T10:15:48.817951+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6283,"log_dups_size":0,"ondisk_log_size":6283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:56:20.507993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411762688,"num_objects":6283,"num_object_clones":0,"num_object_copies":18849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6283,"num_write_kb":402112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":166,"num_bytes_recovered":10878976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":251,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115381+0000","last_change":"2024-09-21T10:13:09.578151+0000","last_active":"2024-09-21T10:15:36.115381+0000","last_peered":"2024-09-21T10:15:36.115381+0000","last_clean":"2024-09-21T10:15:36.115381+0000","last_became_active":"2024-09-21T10:13:09.577233+0000","last_became_peered":"2024-09-21T10:13:09.577233+0000","last_unstale":"2024-09-21T10:15:36.115381+0000","last_undegraded":"2024-09-21T10:15:36.115381+0000","last_fullsized":"2024-09-21T10:15:36.115381+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:00.627590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":27,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118515+0000","last_change":"2024-09-21T10:14:32.771000+0000","last_active":"2024-09-21T10:15:36.118515+0000","last_peered":"2024-09-21T10:15:36.118515+0000","last_clean":"2024-09-21T10:15:36.118515+0000","last_became_active":"2024-09-21T10:14:32.770562+0000","last_became_peered":"2024-09-21T10:14:32.770562+0000","last_unstale":"2024-09-21T10:15:36.118515+0000","last_undegraded":"2024-09-21T10:15:36.118515+0000","last_fullsized":"2024-09-21T10:15:36.118515+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:47.956518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"265'6238","reported_seq":6267,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.597149+0000","last_change":"2024-09-21T10:14:28.441368+0000","last_active":"2024-09-21T10:15:49.597149+0000","last_peered":"2024-09-21T10:15:49.597149+0000","last_clean":"2024-09-21T10:15:49.597149+0000","last_became_active":"2024-09-21T10:14:28.441092+0000","last_became_peered":"2024-09-21T10:14:28.441092+0000","last_unstale":"2024-09-21T10:15:49.597149+0000","last_undegraded":"2024-09-21T10:15:49.597149+0000","last_fullsized":"2024-09-21T10:15:49.597149+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6238,"log_dups_size":0,"ondisk_log_size":6238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:28.336965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408813568,"num_objects":6238,"num_object_clones":0,"num_object_copies":18714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6238,"num_write_kb":399232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":233,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119067+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:15:36.119067+0000","last_peered":"2024-09-21T10:15:36.119067+0000","last_clean":"2024-09-21T10:15:36.119067+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:15:36.119067+0000","last_undegraded":"2024-09-21T10:15:36.119067+0000","last_fullsized":"2024-09-21T10:15:36.119067+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":250,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118727+0000","last_change":"2024-09-21T10:13:09.579444+0000","last_active":"2024-09-21T10:15:36.118727+0000","last_peered":"2024-09-21T10:15:36.118727+0000","last_clean":"2024-09-21T10:15:36.118727+0000","last_became_active":"2024-09-21T10:13:09.579162+0000","last_became_peered":"2024-09-21T10:13:09.579162+0000","last_unstale":"2024-09-21T10:15:36.118727+0000","last_undegraded":"2024-09-21T10:15:36.118727+0000","last_fullsized":"2024-09-21T10:15:36.118727+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.051244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"265'6209","reported_seq":6238,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.622686+0000","last_change":"2024-09-21T10:14:28.439503+0000","last_active":"2024-09-21T10:15:49.622686+0000","last_peered":"2024-09-21T10:15:49.622686+0000","last_clean":"2024-09-21T10:15:49.622686+0000","last_became_active":"2024-09-21T10:14:28.438413+0000","last_became_peered":"2024-09-21T10:14:28.438413+0000","last_unstale":"2024-09-21T10:15:49.622686+0000","last_undegraded":"2024-09-21T10:15:49.622686+0000","last_fullsized":"2024-09-21T10:15:49.622686+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6209,"log_dups_size":0,"ondisk_log_size":6209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:24.386364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406913024,"num_objects":6209,"num_object_clones":0,"num_object_copies":18627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6209,"num_write_kb":397376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":403,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.111934+0000","last_change":"2024-09-21T10:13:09.582151+0000","last_active":"2024-09-21T10:15:36.111934+0000","last_peered":"2024-09-21T10:15:36.111934+0000","last_clean":"2024-09-21T10:15:36.111934+0000","last_became_active":"2024-09-21T10:13:09.580224+0000","last_became_peered":"2024-09-21T10:13:09.580224+0000","last_unstale":"2024-09-21T10:15:36.111934+0000","last_undegraded":"2024-09-21T10:15:36.111934+0000","last_fullsized":"2024-09-21T10:15:36.111934+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:15.490461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.616042+0000","last_change":"2024-09-21T10:13:09.586643+0000","last_active":"2024-09-21T10:15:49.616042+0000","last_peered":"2024-09-21T10:15:49.616042+0000","last_clean":"2024-09-21T10:15:49.616042+0000","last_became_active":"2024-09-21T10:13:09.586283+0000","last_became_peered":"2024-09-21T10:13:09.586283+0000","last_unstale":"2024-09-21T10:15:49.616042+0000","last_undegraded":"2024-09-21T10:15:49.616042+0000","last_fullsized":"2024-09-21T10:15:49.616042+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:20.002373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"265'6346","reported_seq":6705,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.594944+0000","last_change":"2024-09-21T10:14:49.554445+0000","last_active":"2024-09-21T10:15:49.594944+0000","last_peered":"2024-09-21T10:15:49.594944+0000","last_clean":"2024-09-21T10:15:49.594944+0000","last_became_active":"2024-09-21T10:14:49.552633+0000","last_became_peered":"2024-09-21T10:14:49.552633+0000","last_unstale":"2024-09-21T10:15:49.594944+0000","last_undegraded":"2024-09-21T10:15:49.594944+0000","last_fullsized":"2024-09-21T10:15:49.594944+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6346,"log_dups_size":0,"ondisk_log_size":6346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:42.174531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415891456,"num_objects":6346,"num_object_clones":0,"num_object_copies":19038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6346,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":634,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115204+0000","last_change":"2024-09-21T10:13:12.573143+0000","last_active":"2024-09-21T10:15:36.115204+0000","last_peered":"2024-09-21T10:15:36.115204+0000","last_clean":"2024-09-21T10:15:36.115204+0000","last_became_active":"2024-09-21T10:13:12.571751+0000","last_became_peered":"2024-09-21T10:13:12.571751+0000","last_unstale":"2024-09-21T10:15:36.115204+0000","last_undegraded":"2024-09-21T10:15:36.115204+0000","last_fullsized":"2024-09-21T10:15:36.115204+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:28.804119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":583,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112591+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:15:36.112591+0000","last_peered":"2024-09-21T10:15:36.112591+0000","last_clean":"2024-09-21T10:15:36.112591+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:15:36.112591+0000","last_undegraded":"2024-09-21T10:15:36.112591+0000","last_fullsized":"2024-09-21T10:15:36.112591+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":350,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116791+0000","last_change":"2024-09-21T10:14:45.527731+0000","last_active":"2024-09-21T10:15:36.116791+0000","last_peered":"2024-09-21T10:15:36.116791+0000","last_clean":"2024-09-21T10:15:36.116791+0000","last_became_active":"2024-09-21T10:14:45.527330+0000","last_became_peered":"2024-09-21T10:14:45.527330+0000","last_unstale":"2024-09-21T10:15:36.116791+0000","last_undegraded":"2024-09-21T10:15:36.116791+0000","last_fullsized":"2024-09-21T10:15:36.116791+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:03:48.411604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":218,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115873+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:15:36.115873+0000","last_peered":"2024-09-21T10:15:36.115873+0000","last_clean":"2024-09-21T10:15:36.115873+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:15:36.115873+0000","last_undegraded":"2024-09-21T10:15:36.115873+0000","last_fullsized":"2024-09-21T10:15:36.115873+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"265'6238","reported_seq":6267,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.626058+0000","last_change":"2024-09-21T10:14:28.440396+0000","last_active":"2024-09-21T10:15:49.626058+0000","last_peered":"2024-09-21T10:15:49.626058+0000","last_clean":"2024-09-21T10:15:49.626058+0000","last_became_active":"2024-09-21T10:14:28.440137+0000","last_became_peered":"2024-09-21T10:14:28.440137+0000","last_unstale":"2024-09-21T10:15:49.626058+0000","last_undegraded":"2024-09-21T10:15:49.626058+0000","last_fullsized":"2024-09-21T10:15:49.626058+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6238,"log_dups_size":0,"ondisk_log_size":6238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:33:10.456362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408813568,"num_objects":6238,"num_object_clones":0,"num_object_copies":18714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6238,"num_write_kb":399232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":858,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115168+0000","last_change":"2024-09-21T10:13:59.583227+0000","last_active":"2024-09-21T10:15:36.115168+0000","last_peered":"2024-09-21T10:15:36.115168+0000","last_clean":"2024-09-21T10:15:36.115168+0000","last_became_active":"2024-09-21T10:13:59.582623+0000","last_became_peered":"2024-09-21T10:13:59.582623+0000","last_unstale":"2024-09-21T10:15:36.115168+0000","last_undegraded":"2024-09-21T10:15:36.115168+0000","last_fullsized":"2024-09-21T10:15:36.115168+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:35.235896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":27,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118247+0000","last_change":"2024-09-21T10:14:32.461889+0000","last_active":"2024-09-21T10:15:36.118247+0000","last_peered":"2024-09-21T10:15:36.118247+0000","last_clean":"2024-09-21T10:15:36.118247+0000","last_became_active":"2024-09-21T10:14:32.461551+0000","last_became_peered":"2024-09-21T10:14:32.461551+0000","last_unstale":"2024-09-21T10:15:36.118247+0000","last_undegraded":"2024-09-21T10:15:36.118247+0000","last_fullsized":"2024-09-21T10:15:36.118247+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:30:45.261092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.604822+0000","last_change":"2024-09-21T10:13:09.580568+0000","last_active":"2024-09-21T10:15:49.604822+0000","last_peered":"2024-09-21T10:15:49.604822+0000","last_clean":"2024-09-21T10:15:49.604822+0000","last_became_active":"2024-09-21T10:13:09.579053+0000","last_became_peered":"2024-09-21T10:13:09.579053+0000","last_unstale":"2024-09-21T10:15:49.604822+0000","last_undegraded":"2024-09-21T10:15:49.604822+0000","last_fullsized":"2024-09-21T10:15:49.604822+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:39.206612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"264'6242","reported_seq":6269,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.585636+0000","last_change":"2024-09-21T10:14:32.642821+0000","last_active":"2024-09-21T10:15:49.585636+0000","last_peered":"2024-09-21T10:15:49.585636+0000","last_clean":"2024-09-21T10:15:49.585636+0000","last_became_active":"2024-09-21T10:14:32.642404+0000","last_became_peered":"2024-09-21T10:14:32.642404+0000","last_unstale":"2024-09-21T10:15:49.585636+0000","last_undegraded":"2024-09-21T10:15:49.585636+0000","last_fullsized":"2024-09-21T10:15:49.585636+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6242,"log_dups_size":0,"ondisk_log_size":6242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:02.051331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409075712,"num_objects":6242,"num_object_clones":0,"num_object_copies":18726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6242,"num_write_kb":399488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":919,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116040+0000","last_change":"2024-09-21T10:13:59.590250+0000","last_active":"2024-09-21T10:15:36.116040+0000","last_peered":"2024-09-21T10:15:36.116040+0000","last_clean":"2024-09-21T10:15:36.116040+0000","last_became_active":"2024-09-21T10:13:59.587283+0000","last_became_peered":"2024-09-21T10:13:59.587283+0000","last_unstale":"2024-09-21T10:15:36.116040+0000","last_undegraded":"2024-09-21T10:15:36.116040+0000","last_fullsized":"2024-09-21T10:15:36.116040+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:14.118934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.604888+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:15:49.604888+0000","last_peered":"2024-09-21T10:15:49.604888+0000","last_clean":"2024-09-21T10:15:49.604888+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:15:49.604888+0000","last_undegraded":"2024-09-21T10:15:49.604888+0000","last_fullsized":"2024-09-21T10:15:49.604888+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.616121+0000","last_change":"2024-09-21T10:13:09.578776+0000","last_active":"2024-09-21T10:15:49.616121+0000","last_peered":"2024-09-21T10:15:49.616121+0000","last_clean":"2024-09-21T10:15:49.616121+0000","last_became_active":"2024-09-21T10:13:09.578462+0000","last_became_peered":"2024-09-21T10:13:09.578462+0000","last_unstale":"2024-09-21T10:15:49.616121+0000","last_undegraded":"2024-09-21T10:15:49.616121+0000","last_fullsized":"2024-09-21T10:15:49.616121+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:19.035767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"264'6239","reported_seq":6268,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.590342+0000","last_change":"2024-09-21T10:14:28.440111+0000","last_active":"2024-09-21T10:15:49.590342+0000","last_peered":"2024-09-21T10:15:49.590342+0000","last_clean":"2024-09-21T10:15:49.590342+0000","last_became_active":"2024-09-21T10:14:28.439733+0000","last_became_peered":"2024-09-21T10:14:28.439733+0000","last_unstale":"2024-09-21T10:15:49.590342+0000","last_undegraded":"2024-09-21T10:15:49.590342+0000","last_fullsized":"2024-09-21T10:15:49.590342+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6239,"log_dups_size":0,"ondisk_log_size":6239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:17.534207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408879104,"num_objects":6239,"num_object_clones":0,"num_object_copies":18717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6239,"num_write_kb":399296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1057,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112822+0000","last_change":"2024-09-21T10:13:12.560467+0000","last_active":"2024-09-21T10:15:36.112822+0000","last_peered":"2024-09-21T10:15:36.112822+0000","last_clean":"2024-09-21T10:15:36.112822+0000","last_became_active":"2024-09-21T10:13:12.559882+0000","last_became_peered":"2024-09-21T10:13:12.559882+0000","last_unstale":"2024-09-21T10:15:36.112822+0000","last_undegraded":"2024-09-21T10:15:36.112822+0000","last_fullsized":"2024-09-21T10:15:36.112822+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:33.267760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":135,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.117358+0000","last_change":"2024-09-21T10:13:56.570638+0000","last_active":"2024-09-21T10:15:36.117358+0000","last_peered":"2024-09-21T10:15:36.117358+0000","last_clean":"2024-09-21T10:15:36.117358+0000","last_became_active":"2024-09-21T10:13:56.570132+0000","last_became_peered":"2024-09-21T10:13:56.570132+0000","last_unstale":"2024-09-21T10:15:36.117358+0000","last_undegraded":"2024-09-21T10:15:36.117358+0000","last_fullsized":"2024-09-21T10:15:36.117358+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:28.451531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.605412+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:15:49.605412+0000","last_peered":"2024-09-21T10:15:49.605412+0000","last_clean":"2024-09-21T10:15:49.605412+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:15:49.605412+0000","last_undegraded":"2024-09-21T10:15:49.605412+0000","last_fullsized":"2024-09-21T10:15:49.605412+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"265'6178","reported_seq":6207,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.612479+0000","last_change":"2024-09-21T10:14:28.437169+0000","last_active":"2024-09-21T10:15:49.612479+0000","last_peered":"2024-09-21T10:15:49.612479+0000","last_clean":"2024-09-21T10:15:49.612479+0000","last_became_active":"2024-09-21T10:14:28.436873+0000","last_became_peered":"2024-09-21T10:14:28.436873+0000","last_unstale":"2024-09-21T10:15:49.612479+0000","last_undegraded":"2024-09-21T10:15:49.612479+0000","last_fullsized":"2024-09-21T10:15:49.612479+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6178,"log_dups_size":0,"ondisk_log_size":6178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:16.516429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404881408,"num_objects":6178,"num_object_clones":0,"num_object_copies":18534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6178,"num_write_kb":395392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":249,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.113026+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:15:36.113026+0000","last_peered":"2024-09-21T10:15:36.113026+0000","last_clean":"2024-09-21T10:15:36.113026+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:15:36.113026+0000","last_undegraded":"2024-09-21T10:15:36.113026+0000","last_fullsized":"2024-09-21T10:15:36.113026+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":688,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.605626+0000","last_change":"2024-09-21T10:13:59.577297+0000","last_active":"2024-09-21T10:15:49.605626+0000","last_peered":"2024-09-21T10:15:49.605626+0000","last_clean":"2024-09-21T10:15:49.605626+0000","last_became_active":"2024-09-21T10:13:59.576704+0000","last_became_peered":"2024-09-21T10:13:59.576704+0000","last_unstale":"2024-09-21T10:15:49.605626+0000","last_undegraded":"2024-09-21T10:15:49.605626+0000","last_fullsized":"2024-09-21T10:15:49.605626+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:11:28.773044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":125,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.614232+0000","last_change":"2024-09-21T10:14:45.531907+0000","last_active":"2024-09-21T10:15:49.614232+0000","last_peered":"2024-09-21T10:15:49.614232+0000","last_clean":"2024-09-21T10:15:49.614232+0000","last_became_active":"2024-09-21T10:14:45.531536+0000","last_became_peered":"2024-09-21T10:14:45.531536+0000","last_unstale":"2024-09-21T10:15:49.614232+0000","last_undegraded":"2024-09-21T10:15:49.614232+0000","last_fullsized":"2024-09-21T10:15:49.614232+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:18:52.155338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"265'6327","reported_seq":6661,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.625900+0000","last_change":"2024-09-21T10:14:48.563829+0000","last_active":"2024-09-21T10:15:49.625900+0000","last_peered":"2024-09-21T10:15:49.625900+0000","last_clean":"2024-09-21T10:15:49.625900+0000","last_became_active":"2024-09-21T10:14:48.549468+0000","last_became_peered":"2024-09-21T10:14:48.549468+0000","last_unstale":"2024-09-21T10:15:49.625900+0000","last_undegraded":"2024-09-21T10:15:49.625900+0000","last_fullsized":"2024-09-21T10:15:49.625900+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6327,"log_dups_size":0,"ondisk_log_size":6327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:08.229536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414646272,"num_objects":6327,"num_object_clones":0,"num_object_copies":18981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6327,"num_write_kb":404928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":118,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.605539+0000","last_change":"2024-09-21T10:14:45.530390+0000","last_active":"2024-09-21T10:15:49.605539+0000","last_peered":"2024-09-21T10:15:49.605539+0000","last_clean":"2024-09-21T10:15:49.605539+0000","last_became_active":"2024-09-21T10:14:45.530054+0000","last_became_peered":"2024-09-21T10:14:45.530054+0000","last_unstale":"2024-09-21T10:15:49.605539+0000","last_undegraded":"2024-09-21T10:15:49.605539+0000","last_fullsized":"2024-09-21T10:15:49.605539+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:42:49.775110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":746,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116613+0000","last_change":"2024-09-21T10:13:09.604050+0000","last_active":"2024-09-21T10:15:36.116613+0000","last_peered":"2024-09-21T10:15:36.116613+0000","last_clean":"2024-09-21T10:15:36.116613+0000","last_became_active":"2024-09-21T10:13:09.583593+0000","last_became_peered":"2024-09-21T10:13:09.583593+0000","last_unstale":"2024-09-21T10:15:36.116613+0000","last_undegraded":"2024-09-21T10:15:36.116613+0000","last_fullsized":"2024-09-21T10:15:36.116613+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:35.420105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":329,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112577+0000","last_change":"2024-09-21T10:14:45.528988+0000","last_active":"2024-09-21T10:15:36.112577+0000","last_peered":"2024-09-21T10:15:36.112577+0000","last_clean":"2024-09-21T10:15:36.112577+0000","last_became_active":"2024-09-21T10:14:45.528689+0000","last_became_peered":"2024-09-21T10:14:45.528689+0000","last_unstale":"2024-09-21T10:15:36.112577+0000","last_undegraded":"2024-09-21T10:15:36.112577+0000","last_fullsized":"2024-09-21T10:15:36.112577+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:48:43.539523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"264'6239","reported_seq":6287,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:47.604453+0000","last_change":"2024-09-21T10:14:28.438913+0000","last_active":"2024-09-21T10:15:47.604453+0000","last_peered":"2024-09-21T10:15:47.604453+0000","last_clean":"2024-09-21T10:15:47.604453+0000","last_became_active":"2024-09-21T10:14:28.438598+0000","last_became_peered":"2024-09-21T10:14:28.438598+0000","last_unstale":"2024-09-21T10:15:47.604453+0000","last_undegraded":"2024-09-21T10:15:47.604453+0000","last_fullsized":"2024-09-21T10:15:47.604453+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6239,"log_dups_size":0,"ondisk_log_size":6239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:29:24.765631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408879104,"num_objects":6239,"num_object_clones":0,"num_object_copies":18717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6239,"num_write_kb":399296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":397,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112063+0000","last_change":"2024-09-21T10:13:56.647321+0000","last_active":"2024-09-21T10:15:36.112063+0000","last_peered":"2024-09-21T10:15:36.112063+0000","last_clean":"2024-09-21T10:15:36.112063+0000","last_became_active":"2024-09-21T10:13:56.644948+0000","last_became_peered":"2024-09-21T10:13:56.644948+0000","last_unstale":"2024-09-21T10:15:36.112063+0000","last_undegraded":"2024-09-21T10:15:36.112063+0000","last_fullsized":"2024-09-21T10:15:36.112063+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:40.992911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":715,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.605018+0000","last_change":"2024-09-21T10:13:12.569309+0000","last_active":"2024-09-21T10:15:49.605018+0000","last_peered":"2024-09-21T10:15:49.605018+0000","last_clean":"2024-09-21T10:15:49.605018+0000","last_became_active":"2024-09-21T10:13:12.568325+0000","last_became_peered":"2024-09-21T10:13:12.568325+0000","last_unstale":"2024-09-21T10:15:49.605018+0000","last_undegraded":"2024-09-21T10:15:49.605018+0000","last_fullsized":"2024-09-21T10:15:49.605018+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:18.314566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":404,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.111720+0000","last_change":"2024-09-21T10:13:09.579331+0000","last_active":"2024-09-21T10:15:36.111720+0000","last_peered":"2024-09-21T10:15:36.111720+0000","last_clean":"2024-09-21T10:15:36.111720+0000","last_became_active":"2024-09-21T10:13:09.578777+0000","last_became_peered":"2024-09-21T10:13:09.578777+0000","last_unstale":"2024-09-21T10:15:36.111720+0000","last_undegraded":"2024-09-21T10:15:36.111720+0000","last_fullsized":"2024-09-21T10:15:36.111720+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:50:34.425710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"264'6114","reported_seq":6163,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:47.615607+0000","last_change":"2024-09-21T10:14:28.439188+0000","last_active":"2024-09-21T10:15:47.615607+0000","last_peered":"2024-09-21T10:15:47.615607+0000","last_clean":"2024-09-21T10:15:47.615607+0000","last_became_active":"2024-09-21T10:14:28.438763+0000","last_became_peered":"2024-09-21T10:14:28.438763+0000","last_unstale":"2024-09-21T10:15:47.615607+0000","last_undegraded":"2024-09-21T10:15:47.615607+0000","last_fullsized":"2024-09-21T10:15:47.615607+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6114,"log_dups_size":0,"ondisk_log_size":6114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:31.383586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400687104,"num_objects":6114,"num_object_clones":0,"num_object_copies":18342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6114,"num_write_kb":391296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":115,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.604448+0000","last_change":"2024-09-21T10:13:09.592519+0000","last_active":"2024-09-21T10:15:49.604448+0000","last_peered":"2024-09-21T10:15:49.604448+0000","last_clean":"2024-09-21T10:15:49.604448+0000","last_became_active":"2024-09-21T10:13:09.587615+0000","last_became_peered":"2024-09-21T10:13:09.587615+0000","last_unstale":"2024-09-21T10:15:49.604448+0000","last_undegraded":"2024-09-21T10:15:49.604448+0000","last_fullsized":"2024-09-21T10:15:49.604448+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:22:43.216612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":509,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116073+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:15:36.116073+0000","last_peered":"2024-09-21T10:15:36.116073+0000","last_clean":"2024-09-21T10:15:36.116073+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:15:36.116073+0000","last_undegraded":"2024-09-21T10:15:36.116073+0000","last_fullsized":"2024-09-21T10:15:36.116073+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":218,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118934+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:15:36.118934+0000","last_peered":"2024-09-21T10:15:36.118934+0000","last_clean":"2024-09-21T10:15:36.118934+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:15:36.118934+0000","last_undegraded":"2024-09-21T10:15:36.118934+0000","last_fullsized":"2024-09-21T10:15:36.118934+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":258,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118637+0000","last_change":"2024-09-21T10:15:21.038463+0000","last_active":"2024-09-21T10:15:36.118637+0000","last_peered":"2024-09-21T10:15:36.118637+0000","last_clean":"2024-09-21T10:15:36.118637+0000","last_became_active":"2024-09-21T10:15:21.038061+0000","last_became_peered":"2024-09-21T10:15:21.038061+0000","last_unstale":"2024-09-21T10:15:36.118637+0000","last_undegraded":"2024-09-21T10:15:36.118637+0000","last_fullsized":"2024-09-21T10:15:36.118637+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.241295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":240,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119477+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:15:36.119477+0000","last_peered":"2024-09-21T10:15:36.119477+0000","last_clean":"2024-09-21T10:15:36.119477+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:15:36.119477+0000","last_undegraded":"2024-09-21T10:15:36.119477+0000","last_fullsized":"2024-09-21T10:15:36.119477+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112564+0000","last_change":"2024-09-21T10:15:21.039385+0000","last_active":"2024-09-21T10:15:36.112564+0000","last_peered":"2024-09-21T10:15:36.112564+0000","last_clean":"2024-09-21T10:15:36.112564+0000","last_became_active":"2024-09-21T10:15:21.039073+0000","last_became_peered":"2024-09-21T10:15:21.039073+0000","last_unstale":"2024-09-21T10:15:36.112564+0000","last_undegraded":"2024-09-21T10:15:36.112564+0000","last_fullsized":"2024-09-21T10:15:36.112564+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:11.160836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":235,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115796+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:15:36.115796+0000","last_peered":"2024-09-21T10:15:36.115796+0000","last_clean":"2024-09-21T10:15:36.115796+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:15:36.115796+0000","last_undegraded":"2024-09-21T10:15:36.115796+0000","last_fullsized":"2024-09-21T10:15:36.115796+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":23,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112816+0000","last_change":"2024-09-21T10:14:45.528895+0000","last_active":"2024-09-21T10:15:36.112816+0000","last_peered":"2024-09-21T10:15:36.112816+0000","last_clean":"2024-09-21T10:15:36.112816+0000","last_became_active":"2024-09-21T10:14:45.528608+0000","last_became_peered":"2024-09-21T10:14:45.528608+0000","last_unstale":"2024-09-21T10:15:36.112816+0000","last_undegraded":"2024-09-21T10:15:36.112816+0000","last_fullsized":"2024-09-21T10:15:36.112816+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:48.887817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":336,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112077+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:15:36.112077+0000","last_peered":"2024-09-21T10:15:36.112077+0000","last_clean":"2024-09-21T10:15:36.112077+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:15:36.112077+0000","last_undegraded":"2024-09-21T10:15:36.112077+0000","last_fullsized":"2024-09-21T10:15:36.112077+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":23,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112751+0000","last_change":"2024-09-21T10:14:45.530392+0000","last_active":"2024-09-21T10:15:36.112751+0000","last_peered":"2024-09-21T10:15:36.112751+0000","last_clean":"2024-09-21T10:15:36.112751+0000","last_became_active":"2024-09-21T10:14:45.530079+0000","last_became_peered":"2024-09-21T10:14:45.530079+0000","last_unstale":"2024-09-21T10:15:36.112751+0000","last_undegraded":"2024-09-21T10:15:36.112751+0000","last_fullsized":"2024-09-21T10:15:36.112751+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:17.306916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":415,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.111634+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:15:36.111634+0000","last_peered":"2024-09-21T10:15:36.111634+0000","last_clean":"2024-09-21T10:15:36.111634+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:15:36.111634+0000","last_undegraded":"2024-09-21T10:15:36.111634+0000","last_fullsized":"2024-09-21T10:15:36.111634+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":260,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116026+0000","last_change":"2024-09-21T10:13:09.573620+0000","last_active":"2024-09-21T10:15:36.116026+0000","last_peered":"2024-09-21T10:15:36.116026+0000","last_clean":"2024-09-21T10:15:36.116026+0000","last_became_active":"2024-09-21T10:13:09.573216+0000","last_became_peered":"2024-09-21T10:13:09.573216+0000","last_unstale":"2024-09-21T10:15:36.116026+0000","last_undegraded":"2024-09-21T10:15:36.116026+0000","last_fullsized":"2024-09-21T10:15:36.116026+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:40.253863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":424,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.111811+0000","last_change":"2024-09-21T10:13:09.581932+0000","last_active":"2024-09-21T10:15:36.111811+0000","last_peered":"2024-09-21T10:15:36.111811+0000","last_clean":"2024-09-21T10:15:36.111811+0000","last_became_active":"2024-09-21T10:13:09.580798+0000","last_became_peered":"2024-09-21T10:13:09.580798+0000","last_unstale":"2024-09-21T10:15:36.111811+0000","last_undegraded":"2024-09-21T10:15:36.111811+0000","last_fullsized":"2024-09-21T10:15:36.111811+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:05.563826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":109,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115442+0000","last_change":"2024-09-21T10:15:21.033453+0000","last_active":"2024-09-21T10:15:36.115442+0000","last_peered":"2024-09-21T10:15:36.115442+0000","last_clean":"2024-09-21T10:15:36.115442+0000","last_became_active":"2024-09-21T10:15:21.032893+0000","last_became_peered":"2024-09-21T10:15:21.032893+0000","last_unstale":"2024-09-21T10:15:36.115442+0000","last_undegraded":"2024-09-21T10:15:36.115442+0000","last_fullsized":"2024-09-21T10:15:36.115442+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:14.164715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":118,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.604545+0000","last_change":"2024-09-21T10:14:32.454735+0000","last_active":"2024-09-21T10:15:49.604545+0000","last_peered":"2024-09-21T10:15:49.604545+0000","last_clean":"2024-09-21T10:15:49.604545+0000","last_became_active":"2024-09-21T10:14:32.454475+0000","last_became_peered":"2024-09-21T10:14:32.454475+0000","last_unstale":"2024-09-21T10:15:49.604545+0000","last_undegraded":"2024-09-21T10:15:49.604545+0000","last_fullsized":"2024-09-21T10:15:49.604545+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:59.191649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":407,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.117103+0000","last_change":"2024-09-21T10:14:20.761163+0000","last_active":"2024-09-21T10:15:36.117103+0000","last_peered":"2024-09-21T10:15:36.117103+0000","last_clean":"2024-09-21T10:15:36.117103+0000","last_became_active":"2024-09-21T10:14:20.760814+0000","last_became_peered":"2024-09-21T10:14:20.760814+0000","last_unstale":"2024-09-21T10:15:36.117103+0000","last_undegraded":"2024-09-21T10:15:36.117103+0000","last_fullsized":"2024-09-21T10:15:36.117103+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:44.679743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":266,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119161+0000","last_change":"2024-09-21T10:13:09.577994+0000","last_active":"2024-09-21T10:15:36.119161+0000","last_peered":"2024-09-21T10:15:36.119161+0000","last_clean":"2024-09-21T10:15:36.119161+0000","last_became_active":"2024-09-21T10:13:09.577311+0000","last_became_peered":"2024-09-21T10:13:09.577311+0000","last_unstale":"2024-09-21T10:15:36.119161+0000","last_undegraded":"2024-09-21T10:15:36.119161+0000","last_fullsized":"2024-09-21T10:15:36.119161+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:27.532202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":421,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116263+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:15:36.116263+0000","last_peered":"2024-09-21T10:15:36.116263+0000","last_clean":"2024-09-21T10:15:36.116263+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:15:36.116263+0000","last_undegraded":"2024-09-21T10:15:36.116263+0000","last_fullsized":"2024-09-21T10:15:36.116263+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":145,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118642+0000","last_change":"2024-09-21T10:13:09.578314+0000","last_active":"2024-09-21T10:15:36.118642+0000","last_peered":"2024-09-21T10:15:36.118642+0000","last_clean":"2024-09-21T10:15:36.118642+0000","last_became_active":"2024-09-21T10:13:09.577570+0000","last_became_peered":"2024-09-21T10:13:09.577570+0000","last_unstale":"2024-09-21T10:15:36.118642+0000","last_undegraded":"2024-09-21T10:15:36.118642+0000","last_fullsized":"2024-09-21T10:15:36.118642+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:21.299713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":111,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118541+0000","last_change":"2024-09-21T10:13:09.577841+0000","last_active":"2024-09-21T10:15:36.118541+0000","last_peered":"2024-09-21T10:15:36.118541+0000","last_clean":"2024-09-21T10:15:36.118541+0000","last_became_active":"2024-09-21T10:13:09.576793+0000","last_became_peered":"2024-09-21T10:13:09.576793+0000","last_unstale":"2024-09-21T10:15:36.118541+0000","last_undegraded":"2024-09-21T10:15:36.118541+0000","last_fullsized":"2024-09-21T10:15:36.118541+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:12.718520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":260,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112593+0000","last_change":"2024-09-21T10:13:09.589654+0000","last_active":"2024-09-21T10:15:36.112593+0000","last_peered":"2024-09-21T10:15:36.112593+0000","last_clean":"2024-09-21T10:15:36.112593+0000","last_became_active":"2024-09-21T10:13:09.574438+0000","last_became_peered":"2024-09-21T10:13:09.574438+0000","last_unstale":"2024-09-21T10:15:36.112593+0000","last_undegraded":"2024-09-21T10:15:36.112593+0000","last_fullsized":"2024-09-21T10:15:36.112593+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:12.504161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":27,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118168+0000","last_change":"2024-09-21T10:14:32.771090+0000","last_active":"2024-09-21T10:15:36.118168+0000","last_peered":"2024-09-21T10:15:36.118168+0000","last_clean":"2024-09-21T10:15:36.118168+0000","last_became_active":"2024-09-21T10:14:32.770619+0000","last_became_peered":"2024-09-21T10:14:32.770619+0000","last_unstale":"2024-09-21T10:15:36.118168+0000","last_undegraded":"2024-09-21T10:15:36.118168+0000","last_fullsized":"2024-09-21T10:15:36.118168+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:29:31.473095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":219,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.113004+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:15:36.113004+0000","last_peered":"2024-09-21T10:15:36.113004+0000","last_clean":"2024-09-21T10:15:36.113004+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:15:36.113004+0000","last_undegraded":"2024-09-21T10:15:36.113004+0000","last_fullsized":"2024-09-21T10:15:36.113004+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":427,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112397+0000","last_change":"2024-09-21T10:14:45.526997+0000","last_active":"2024-09-21T10:15:36.112397+0000","last_peered":"2024-09-21T10:15:36.112397+0000","last_clean":"2024-09-21T10:15:36.112397+0000","last_became_active":"2024-09-21T10:14:45.526600+0000","last_became_peered":"2024-09-21T10:14:45.526600+0000","last_unstale":"2024-09-21T10:15:36.112397+0000","last_undegraded":"2024-09-21T10:15:36.112397+0000","last_fullsized":"2024-09-21T10:15:36.112397+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:33:20.585859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":395,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115082+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:15:36.115082+0000","last_peered":"2024-09-21T10:15:36.115082+0000","last_clean":"2024-09-21T10:15:36.115082+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:15:36.115082+0000","last_undegraded":"2024-09-21T10:15:36.115082+0000","last_fullsized":"2024-09-21T10:15:36.115082+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":249,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119000+0000","last_change":"2024-09-21T10:13:09.577388+0000","last_active":"2024-09-21T10:15:36.119000+0000","last_peered":"2024-09-21T10:15:36.119000+0000","last_clean":"2024-09-21T10:15:36.119000+0000","last_became_active":"2024-09-21T10:13:09.577027+0000","last_became_peered":"2024-09-21T10:13:09.577027+0000","last_unstale":"2024-09-21T10:15:36.119000+0000","last_undegraded":"2024-09-21T10:15:36.119000+0000","last_fullsized":"2024-09-21T10:15:36.119000+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:49.522609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":29,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112659+0000","last_change":"2024-09-21T10:14:45.536046+0000","last_active":"2024-09-21T10:15:36.112659+0000","last_peered":"2024-09-21T10:15:36.112659+0000","last_clean":"2024-09-21T10:15:36.112659+0000","last_became_active":"2024-09-21T10:14:45.530295+0000","last_became_peered":"2024-09-21T10:14:45.530295+0000","last_unstale":"2024-09-21T10:15:36.112659+0000","last_undegraded":"2024-09-21T10:15:36.112659+0000","last_fullsized":"2024-09-21T10:15:36.112659+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:34.920194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6539771904,"num_objects":99789,"num_object_clones":0,"num_object_copies":299367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":99789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":99789,"num_write_kb":6386496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19625738240,"data_stored":19625738240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99589,"ondisk_log_size":99589,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":1370,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":12582950,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234314,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2840796,"kb_used_data":2400340,"kb_used_omap":1,"kb_used_meta":440446,"kb_avail":90904356,"statfs":{"total":95995035648,"available":93086060544,"internally_reserved":0,"allocated":2457948160,"data_stored":2456702513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":451017163},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495952,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2763676,"kb_used_data":2375700,"kb_used_omap":1,"kb_used_meta":387966,"kb_avail":90981476,"statfs":{"total":95995035648,"available":93165031424,"internally_reserved":0,"allocated":2432716800,"data_stored":2431471939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":397277639},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757589,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2356556,"kb_used_data":2004676,"kb_used_omap":1,"kb_used_meta":351870,"kb_avail":91388596,"statfs":{"total":95995035648,"available":93581922304,"internally_reserved":0,"allocated":2052788224,"data_stored":2051554447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":360315334},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019225,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2789912,"kb_used_data":2407696,"kb_used_omap":1,"kb_used_meta":382206,"kb_avail":90955240,"statfs":{"total":95995035648,"available":93138165760,"internally_reserved":0,"allocated":2465480704,"data_stored":2464238870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":391379398},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435409,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4786884,"kb_used_data":4308156,"kb_used_omap":5,"kb_used_meta":478714,"kb_avail":88958268,"statfs":{"total":95995035648,"available":91093266432,"internally_reserved":0,"allocated":4411551744,"data_stored":4410325590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":490203176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,4,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598936,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2902084,"kb_used_data":2392828,"kb_used_omap":7,"kb_used_meta":509240,"kb_avail":90843068,"statfs":{"total":95995035648,"available":93023301632,"internally_reserved":0,"allocated":2450255872,"data_stored":2449029936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":521462221},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762463,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3570568,"kb_used_data":3201536,"kb_used_omap":7,"kb_used_meta":369016,"kb_avail":90174584,"statfs":{"total":95995035648,"available":92338774016,"internally_reserved":0,"allocated":3278372864,"data_stored":3277142836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":377872637},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":403726925989,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3287875584,"data_stored":3287875584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2452684800,"data_stored":2452684800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4435804160,"data_stored":4435804160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2465857536,"data_stored":2465857536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2066743296,"data_stored":2066743296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2449408000,"data_stored":2449408000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2467364864,"data_stored":2467364864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:15:53.026 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:15:53.026 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:15:53.026 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:15:53.026 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:15:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:52 smithi045 ceph-mon[95976]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:53.086 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:53.094 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:53.124 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:53.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:53.225 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:53.256 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:53.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:53.358 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:53.429 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:53.530 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:53.566 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:53.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:53.667 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:53.700 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:53.802 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:53.840 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:53.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:53.858 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 70 16 104673 104657 93.4303 65.9375 0.0134774 0.0106982 2024-09-21T10:15:53.880 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:53.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:53.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:53 smithi032 ceph-mon[105748]: pgmap v765: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:15:53.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:53 smithi032 ceph-mon[101299]: pgmap v765: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:15:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.002 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:54.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:53 smithi045 ceph-mon[95976]: pgmap v765: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:15:54.102 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:54.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:15:54.237 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:54.267 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:54.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:54.369 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:54.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.417 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:54.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:54.518 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:54.561 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:54.662 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:54.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.720 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:54.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:54.820 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:54.851 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:54.858 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 71 16 105729 105713 93.0439 66 0.00834066 0.0107428 2024-09-21T10:15:54.952 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:54.986 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:54.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:55.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:55.135 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:55.137 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:55.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:55.239 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:55.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.291 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:55.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:55.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:55.452 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:55.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:55.554 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:55.591 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:55.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:55.692 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:55.726 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:55.795 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:55 smithi032 ceph-mon[101299]: pgmap v766: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:15:55.795 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:55 smithi032 ceph-mon[105748]: pgmap v766: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:15:55.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:55.859 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 72 16 106796 106780 92.6765 66.6875 0.0170803 0.0107852 2024-09-21T10:15:55.869 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:55.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:55.971 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:56.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:55 smithi045 ceph-mon[95976]: pgmap v766: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:15:56.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:56.148 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:56.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:56.249 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:56.300 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:56.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:56.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:56.439 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:56.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:56.541 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:56.581 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:56.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:56.682 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:56.726 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:56.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:56.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:56.859 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 73 16 107782 107766 92.251 61.625 0.0172908 0.0108349 2024-09-21T10:15:56.864 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:56.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:56.880 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3437836142' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:56.880 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3437836142' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:56.966 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:56.995 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3437836142' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:57.096 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:57.132 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:57.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:57.229 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:15:57.229 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:15:57.229 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:57.230 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:57.231 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:57.231 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:57.234 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:57.269 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:57.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:57.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:57.402 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:57.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:57.504 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:57.540 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:57.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:57.642 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:57.676 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:57.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:57.778 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:57.819 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:57.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:57.859 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 74 16 108892 108876 91.9418 69.375 0.0216076 0.0108718 2024-09-21T10:15:57.918 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:57 smithi032 ceph-mon[101299]: pgmap v767: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-21T10:15:57.918 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:57 smithi032 ceph-mon[105748]: pgmap v767: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-21T10:15:57.921 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:57.948 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:57.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:57 smithi045 ceph-mon[95976]: pgmap v767: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-21T10:15:58.050 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:58.079 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:58.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:58.180 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:58.220 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:58.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:58.321 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:58.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:58.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:58.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:58.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:58.602 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:58.625 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:58.625 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:58.630 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:58.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:58.732 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:58.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.769 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:58.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:58.860 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 75 16 109886 109870 91.5442 62.125 0.0067503 0.0109176 2024-09-21T10:15:58.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:58.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.914 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:58.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:59.015 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:59.045 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:59.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:59.055 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:59.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:59.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:59.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:59.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:59.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:59.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:59.476 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:15:59.477 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:15:59.481 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:59.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:59.583 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:59.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.616 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:59.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:59.717 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:59.755 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:15:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:59.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:59.860 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 76 16 110888 110872 91.1636 62.625 0.0105499 0.0109647 2024-09-21T10:15:59.891 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:15:59.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:59.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:15:59 smithi032 ceph-mon[105748]: pgmap v768: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:15:59.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:15:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:15:59] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-09-21T10:15:59.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:15:59 smithi032 ceph-mon[101299]: pgmap v768: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:15:59.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:00.027 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:00.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:15:59 smithi045 ceph-mon[95976]: pgmap v768: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:16:00.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:16:00.129 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:00.158 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:00.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:00.289 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:00.390 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:00.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.442 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:00.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:00.543 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:00.580 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:00.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:00.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:00.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:00.822 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.860 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:00.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:00.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 77 16 111829 111813 90.7428 58.8125 0.0185611 0.0110144 2024-09-21T10:16:00.906 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:00 smithi032 ceph-mon[101299]: from='client.55024 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:16:00.906 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:00 smithi032 ceph-mon[105748]: from='client.55024 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:16:00.961 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:00.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:00.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:01.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:00 smithi045 ceph-mon[95976]: from='client.55024 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:16:01.089 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:01.119 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:01.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:01.221 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:01.254 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:01.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:01.356 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:01.391 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:01.493 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:01.528 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:01.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:01.629 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:01.662 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:01.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:01.764 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:01.786 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:01.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:01.860 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 78 16 112802 112786 90.3591 60.8125 0.00821578 0.0110619 2024-09-21T10:16:01.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:01.925 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:01.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:01 smithi032 ceph-mon[101299]: pgmap v769: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:16:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:01 smithi032 ceph-mon[105748]: pgmap v769: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:16:02.026 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:01 smithi045 ceph-mon[95976]: pgmap v769: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:16:02.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.062 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:02.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:02.163 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:02.204 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:02.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:02.306 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:02.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:02.448 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:02.486 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:02.588 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:02.620 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:02.707 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":768,"stamp":"2024-09-21T10:15:58.377255+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6966417381,"num_objects":106254,"num_object_clones":0,"num_object_copies":318762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106254,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":107349,"num_write_kb":6813505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":927,"num_bytes_recovered":65475136,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106008,"ondisk_log_size":106008,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":23235584,"kb_used_data":20310408,"kb_used_omap":27,"kb_used_meta":2925092,"kb_avail":632980480,"statfs":{"total":671965249536,"available":648172011520,"internally_reserved":0,"allocated":20797857792,"data_stored":20789181947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28144,"internal_metadata":2995294736},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,11,4,7],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":685113344,"num_objects":10454,"num_object_clones":0,"num_object_copies":31362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10454,"num_write_kb":669056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002755"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":404,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.609882+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:15:49.609882+0000","last_peered":"2024-09-21T10:15:49.609882+0000","last_clean":"2024-09-21T10:15:49.609882+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:15:49.609882+0000","last_undegraded":"2024-09-21T10:15:49.609882+0000","last_fullsized":"2024-09-21T10:15:49.609882+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":234,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115552+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:15:36.115552+0000","last_peered":"2024-09-21T10:15:36.115552+0000","last_clean":"2024-09-21T10:15:36.115552+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:15:36.115552+0000","last_undegraded":"2024-09-21T10:15:36.115552+0000","last_fullsized":"2024-09-21T10:15:36.115552+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":23,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112922+0000","last_change":"2024-09-21T10:14:45.529072+0000","last_active":"2024-09-21T10:15:36.112922+0000","last_peered":"2024-09-21T10:15:36.112922+0000","last_clean":"2024-09-21T10:15:36.112922+0000","last_became_active":"2024-09-21T10:14:45.528270+0000","last_became_peered":"2024-09-21T10:14:45.528270+0000","last_unstale":"2024-09-21T10:15:36.112922+0000","last_undegraded":"2024-09-21T10:15:36.112922+0000","last_fullsized":"2024-09-21T10:15:36.112922+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:00.326719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115256+0000","last_change":"2024-09-21T10:13:09.583319+0000","last_active":"2024-09-21T10:15:36.115256+0000","last_peered":"2024-09-21T10:15:36.115256+0000","last_clean":"2024-09-21T10:15:36.115256+0000","last_became_active":"2024-09-21T10:13:09.582953+0000","last_became_peered":"2024-09-21T10:13:09.582953+0000","last_unstale":"2024-09-21T10:15:36.115256+0000","last_undegraded":"2024-09-21T10:15:36.115256+0000","last_fullsized":"2024-09-21T10:15:36.115256+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:13.630031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":231,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.617053+0000","last_change":"2024-09-21T10:09:35.958207+0000","last_active":"2024-09-21T10:15:49.617053+0000","last_peered":"2024-09-21T10:15:49.617053+0000","last_clean":"2024-09-21T10:15:49.617053+0000","last_became_active":"2024-09-21T10:09:35.957860+0000","last_became_peered":"2024-09-21T10:09:35.957860+0000","last_unstale":"2024-09-21T10:15:49.617053+0000","last_undegraded":"2024-09-21T10:15:49.617053+0000","last_fullsized":"2024-09-21T10:15:49.617053+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:37.767930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":217,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118062+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:15:36.118062+0000","last_peered":"2024-09-21T10:15:36.118062+0000","last_clean":"2024-09-21T10:15:36.118062+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:15:36.118062+0000","last_undegraded":"2024-09-21T10:15:36.118062+0000","last_fullsized":"2024-09-21T10:15:36.118062+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":411,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.603912+0000","last_change":"2024-09-21T10:14:20.761360+0000","last_active":"2024-09-21T10:15:49.603912+0000","last_peered":"2024-09-21T10:15:49.603912+0000","last_clean":"2024-09-21T10:15:49.603912+0000","last_became_active":"2024-09-21T10:14:20.760941+0000","last_became_peered":"2024-09-21T10:14:20.760941+0000","last_unstale":"2024-09-21T10:15:49.603912+0000","last_undegraded":"2024-09-21T10:15:49.603912+0000","last_fullsized":"2024-09-21T10:15:49.603912+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:27.842256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":282,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115324+0000","last_change":"2024-09-21T10:14:45.537300+0000","last_active":"2024-09-21T10:15:36.115324+0000","last_peered":"2024-09-21T10:15:36.115324+0000","last_clean":"2024-09-21T10:15:36.115324+0000","last_became_active":"2024-09-21T10:14:45.527447+0000","last_became_peered":"2024-09-21T10:14:45.527447+0000","last_unstale":"2024-09-21T10:15:36.115324+0000","last_undegraded":"2024-09-21T10:15:36.115324+0000","last_fullsized":"2024-09-21T10:15:36.115324+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:33.652169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"265'6637","reported_seq":6976,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.803114+0000","last_change":"2024-09-21T10:14:49.554195+0000","last_active":"2024-09-21T10:15:54.803114+0000","last_peered":"2024-09-21T10:15:54.803114+0000","last_clean":"2024-09-21T10:15:54.803114+0000","last_became_active":"2024-09-21T10:14:49.551887+0000","last_became_peered":"2024-09-21T10:14:49.551887+0000","last_unstale":"2024-09-21T10:15:54.803114+0000","last_undegraded":"2024-09-21T10:15:54.803114+0000","last_fullsized":"2024-09-21T10:15:54.803114+0000","mapping_epoch":256,"log_start":"257'500","ondisk_log_start":"257'500","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6137,"log_dups_size":0,"ondisk_log_size":6137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:26.931226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":434962432,"num_objects":6637,"num_object_clones":0,"num_object_copies":19911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6637,"num_write_kb":424768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":9043968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":273,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116402+0000","last_change":"2024-09-21T10:13:09.586457+0000","last_active":"2024-09-21T10:15:36.116402+0000","last_peered":"2024-09-21T10:15:36.116402+0000","last_clean":"2024-09-21T10:15:36.116402+0000","last_became_active":"2024-09-21T10:13:09.574624+0000","last_became_peered":"2024-09-21T10:13:09.574624+0000","last_unstale":"2024-09-21T10:15:36.116402+0000","last_undegraded":"2024-09-21T10:15:36.116402+0000","last_fullsized":"2024-09-21T10:15:36.116402+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:39:00.778086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":23,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112412+0000","last_change":"2024-09-21T10:14:45.530298+0000","last_active":"2024-09-21T10:15:36.112412+0000","last_peered":"2024-09-21T10:15:36.112412+0000","last_clean":"2024-09-21T10:15:36.112412+0000","last_became_active":"2024-09-21T10:14:45.529912+0000","last_became_peered":"2024-09-21T10:14:45.529912+0000","last_unstale":"2024-09-21T10:15:36.112412+0000","last_undegraded":"2024-09-21T10:15:36.112412+0000","last_fullsized":"2024-09-21T10:15:36.112412+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:50.567684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"265'6811","reported_seq":7181,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:57.663045+0000","last_change":"2024-09-21T10:14:49.553650+0000","last_active":"2024-09-21T10:15:57.663045+0000","last_peered":"2024-09-21T10:15:57.663045+0000","last_clean":"2024-09-21T10:15:57.663045+0000","last_became_active":"2024-09-21T10:14:49.552007+0000","last_became_peered":"2024-09-21T10:14:49.552007+0000","last_unstale":"2024-09-21T10:15:57.663045+0000","last_undegraded":"2024-09-21T10:15:57.663045+0000","last_fullsized":"2024-09-21T10:15:57.663045+0000","mapping_epoch":256,"log_start":"251'100","ondisk_log_start":"251'100","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:43:27.336777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446365696,"num_objects":6811,"num_object_clones":0,"num_object_copies":20433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6811,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":300,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115506+0000","last_change":"2024-09-21T10:13:09.596445+0000","last_active":"2024-09-21T10:15:36.115506+0000","last_peered":"2024-09-21T10:15:36.115506+0000","last_clean":"2024-09-21T10:15:36.115506+0000","last_became_active":"2024-09-21T10:13:09.582665+0000","last_became_peered":"2024-09-21T10:13:09.582665+0000","last_unstale":"2024-09-21T10:15:36.115506+0000","last_undegraded":"2024-09-21T10:15:36.115506+0000","last_fullsized":"2024-09-21T10:15:36.115506+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:23:47.510351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":265,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118857+0000","last_change":"2024-09-21T10:13:09.578819+0000","last_active":"2024-09-21T10:15:36.118857+0000","last_peered":"2024-09-21T10:15:36.118857+0000","last_clean":"2024-09-21T10:15:36.118857+0000","last_became_active":"2024-09-21T10:13:09.576707+0000","last_became_peered":"2024-09-21T10:13:09.576707+0000","last_unstale":"2024-09-21T10:15:36.118857+0000","last_undegraded":"2024-09-21T10:15:36.118857+0000","last_fullsized":"2024-09-21T10:15:36.118857+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:19.175846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"265'6653","reported_seq":6711,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.840409+0000","last_change":"2024-09-21T10:14:32.771740+0000","last_active":"2024-09-21T10:15:54.840409+0000","last_peered":"2024-09-21T10:15:54.840409+0000","last_clean":"2024-09-21T10:15:54.840409+0000","last_became_active":"2024-09-21T10:14:32.771298+0000","last_became_peered":"2024-09-21T10:14:32.771298+0000","last_unstale":"2024-09-21T10:15:54.840409+0000","last_undegraded":"2024-09-21T10:15:54.840409+0000","last_fullsized":"2024-09-21T10:15:54.840409+0000","mapping_epoch":248,"log_start":"257'500","ondisk_log_start":"257'500","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6153,"log_dups_size":0,"ondisk_log_size":6153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:27.287728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436011008,"num_objects":6653,"num_object_clones":0,"num_object_copies":19959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6653,"num_write_kb":425792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":245,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116519+0000","last_change":"2024-09-21T10:14:45.533171+0000","last_active":"2024-09-21T10:15:36.116519+0000","last_peered":"2024-09-21T10:15:36.116519+0000","last_clean":"2024-09-21T10:15:36.116519+0000","last_became_active":"2024-09-21T10:14:45.527709+0000","last_became_peered":"2024-09-21T10:14:45.527709+0000","last_unstale":"2024-09-21T10:15:36.116519+0000","last_undegraded":"2024-09-21T10:15:36.116519+0000","last_fullsized":"2024-09-21T10:15:36.116519+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:33.989738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":250,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115248+0000","last_change":"2024-09-21T10:13:09.578358+0000","last_active":"2024-09-21T10:15:36.115248+0000","last_peered":"2024-09-21T10:15:36.115248+0000","last_clean":"2024-09-21T10:15:36.115248+0000","last_became_active":"2024-09-21T10:13:09.577273+0000","last_became_peered":"2024-09-21T10:13:09.577273+0000","last_unstale":"2024-09-21T10:15:36.115248+0000","last_undegraded":"2024-09-21T10:15:36.115248+0000","last_fullsized":"2024-09-21T10:15:36.115248+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:17.971932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"265'6520","reported_seq":6571,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:53.829067+0000","last_change":"2024-09-21T10:14:28.438058+0000","last_active":"2024-09-21T10:15:53.829067+0000","last_peered":"2024-09-21T10:15:53.829067+0000","last_clean":"2024-09-21T10:15:53.829067+0000","last_became_active":"2024-09-21T10:14:28.437772+0000","last_became_peered":"2024-09-21T10:14:28.437772+0000","last_unstale":"2024-09-21T10:15:53.829067+0000","last_undegraded":"2024-09-21T10:15:53.829067+0000","last_fullsized":"2024-09-21T10:15:53.829067+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6520,"log_dups_size":0,"ondisk_log_size":6520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:14.437061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427294720,"num_objects":6520,"num_object_clones":0,"num_object_copies":19560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6520,"num_write_kb":417280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":233,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112701+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:15:36.112701+0000","last_peered":"2024-09-21T10:15:36.112701+0000","last_clean":"2024-09-21T10:15:36.112701+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:15:36.112701+0000","last_undegraded":"2024-09-21T10:15:36.112701+0000","last_fullsized":"2024-09-21T10:15:36.112701+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":363,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.606024+0000","last_change":"2024-09-21T10:13:56.569595+0000","last_active":"2024-09-21T10:15:49.606024+0000","last_peered":"2024-09-21T10:15:49.606024+0000","last_clean":"2024-09-21T10:15:49.606024+0000","last_became_active":"2024-09-21T10:13:56.567665+0000","last_became_peered":"2024-09-21T10:13:56.567665+0000","last_unstale":"2024-09-21T10:15:49.606024+0000","last_undegraded":"2024-09-21T10:15:49.606024+0000","last_fullsized":"2024-09-21T10:15:49.606024+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:12:35.947882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"265'6802","reported_seq":6853,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:57.680590+0000","last_change":"2024-09-21T10:14:28.441282+0000","last_active":"2024-09-21T10:15:57.680590+0000","last_peered":"2024-09-21T10:15:57.680590+0000","last_clean":"2024-09-21T10:15:57.680590+0000","last_became_active":"2024-09-21T10:14:28.440767+0000","last_became_peered":"2024-09-21T10:14:28.440767+0000","last_unstale":"2024-09-21T10:15:57.680590+0000","last_undegraded":"2024-09-21T10:15:57.680590+0000","last_fullsized":"2024-09-21T10:15:57.680590+0000","mapping_epoch":245,"log_start":"251'100","ondisk_log_start":"251'100","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:56:18.651559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445775872,"num_objects":6802,"num_object_clones":0,"num_object_copies":20406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6802,"num_write_kb":435328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":421,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.606569+0000","last_change":"2024-09-21T10:13:09.594302+0000","last_active":"2024-09-21T10:15:49.606569+0000","last_peered":"2024-09-21T10:15:49.606569+0000","last_clean":"2024-09-21T10:15:49.606569+0000","last_became_active":"2024-09-21T10:13:09.576948+0000","last_became_peered":"2024-09-21T10:13:09.576948+0000","last_unstale":"2024-09-21T10:15:49.606569+0000","last_undegraded":"2024-09-21T10:15:49.606569+0000","last_fullsized":"2024-09-21T10:15:49.606569+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:10.360060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":232,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119317+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:15:36.119317+0000","last_peered":"2024-09-21T10:15:36.119317+0000","last_clean":"2024-09-21T10:15:36.119317+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:15:36.119317+0000","last_undegraded":"2024-09-21T10:15:36.119317+0000","last_fullsized":"2024-09-21T10:15:36.119317+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"265'6599","reported_seq":6995,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:53.796123+0000","last_change":"2024-09-21T10:14:49.553783+0000","last_active":"2024-09-21T10:15:53.796123+0000","last_peered":"2024-09-21T10:15:53.796123+0000","last_clean":"2024-09-21T10:15:53.796123+0000","last_became_active":"2024-09-21T10:14:49.552803+0000","last_became_peered":"2024-09-21T10:14:49.552803+0000","last_unstale":"2024-09-21T10:15:53.796123+0000","last_undegraded":"2024-09-21T10:15:53.796123+0000","last_fullsized":"2024-09-21T10:15:53.796123+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6599,"log_dups_size":0,"ondisk_log_size":6599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:56:20.507993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432472064,"num_objects":6599,"num_object_clones":0,"num_object_copies":19797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6599,"num_write_kb":422336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":166,"num_bytes_recovered":10878976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":251,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115381+0000","last_change":"2024-09-21T10:13:09.578151+0000","last_active":"2024-09-21T10:15:36.115381+0000","last_peered":"2024-09-21T10:15:36.115381+0000","last_clean":"2024-09-21T10:15:36.115381+0000","last_became_active":"2024-09-21T10:13:09.577233+0000","last_became_peered":"2024-09-21T10:13:09.577233+0000","last_unstale":"2024-09-21T10:15:36.115381+0000","last_undegraded":"2024-09-21T10:15:36.115381+0000","last_fullsized":"2024-09-21T10:15:36.115381+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:00.627590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":27,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118515+0000","last_change":"2024-09-21T10:14:32.771000+0000","last_active":"2024-09-21T10:15:36.118515+0000","last_peered":"2024-09-21T10:15:36.118515+0000","last_clean":"2024-09-21T10:15:36.118515+0000","last_became_active":"2024-09-21T10:14:32.770562+0000","last_became_peered":"2024-09-21T10:14:32.770562+0000","last_unstale":"2024-09-21T10:15:36.118515+0000","last_undegraded":"2024-09-21T10:15:36.118515+0000","last_fullsized":"2024-09-21T10:15:36.118515+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:47.956518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"265'6564","reported_seq":6593,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.690840+0000","last_change":"2024-09-21T10:14:28.441368+0000","last_active":"2024-09-21T10:15:54.690840+0000","last_peered":"2024-09-21T10:15:54.690840+0000","last_clean":"2024-09-21T10:15:54.690840+0000","last_became_active":"2024-09-21T10:14:28.441092+0000","last_became_peered":"2024-09-21T10:14:28.441092+0000","last_unstale":"2024-09-21T10:15:54.690840+0000","last_undegraded":"2024-09-21T10:15:54.690840+0000","last_fullsized":"2024-09-21T10:15:54.690840+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6564,"log_dups_size":0,"ondisk_log_size":6564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:28.336965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430178304,"num_objects":6564,"num_object_clones":0,"num_object_copies":19692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6564,"num_write_kb":420096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":233,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119067+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:15:36.119067+0000","last_peered":"2024-09-21T10:15:36.119067+0000","last_clean":"2024-09-21T10:15:36.119067+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:15:36.119067+0000","last_undegraded":"2024-09-21T10:15:36.119067+0000","last_fullsized":"2024-09-21T10:15:36.119067+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":250,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118727+0000","last_change":"2024-09-21T10:13:09.579444+0000","last_active":"2024-09-21T10:15:36.118727+0000","last_peered":"2024-09-21T10:15:36.118727+0000","last_clean":"2024-09-21T10:15:36.118727+0000","last_became_active":"2024-09-21T10:13:09.579162+0000","last_became_peered":"2024-09-21T10:13:09.579162+0000","last_unstale":"2024-09-21T10:15:36.118727+0000","last_undegraded":"2024-09-21T10:15:36.118727+0000","last_fullsized":"2024-09-21T10:15:36.118727+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.051244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"265'6513","reported_seq":6542,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.666559+0000","last_change":"2024-09-21T10:14:28.439503+0000","last_active":"2024-09-21T10:15:54.666559+0000","last_peered":"2024-09-21T10:15:54.666559+0000","last_clean":"2024-09-21T10:15:54.666559+0000","last_became_active":"2024-09-21T10:14:28.438413+0000","last_became_peered":"2024-09-21T10:14:28.438413+0000","last_unstale":"2024-09-21T10:15:54.666559+0000","last_undegraded":"2024-09-21T10:15:54.666559+0000","last_fullsized":"2024-09-21T10:15:54.666559+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6513,"log_dups_size":0,"ondisk_log_size":6513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:24.386364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426835968,"num_objects":6513,"num_object_clones":0,"num_object_copies":19539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6513,"num_write_kb":416832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":405,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.601645+0000","last_change":"2024-09-21T10:13:09.582151+0000","last_active":"2024-09-21T10:15:49.601645+0000","last_peered":"2024-09-21T10:15:49.601645+0000","last_clean":"2024-09-21T10:15:49.601645+0000","last_became_active":"2024-09-21T10:13:09.580224+0000","last_became_peered":"2024-09-21T10:13:09.580224+0000","last_unstale":"2024-09-21T10:15:49.601645+0000","last_undegraded":"2024-09-21T10:15:49.601645+0000","last_fullsized":"2024-09-21T10:15:49.601645+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:15.490461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.616042+0000","last_change":"2024-09-21T10:13:09.586643+0000","last_active":"2024-09-21T10:15:49.616042+0000","last_peered":"2024-09-21T10:15:49.616042+0000","last_clean":"2024-09-21T10:15:49.616042+0000","last_became_active":"2024-09-21T10:13:09.586283+0000","last_became_peered":"2024-09-21T10:13:09.586283+0000","last_unstale":"2024-09-21T10:15:49.616042+0000","last_undegraded":"2024-09-21T10:15:49.616042+0000","last_fullsized":"2024-09-21T10:15:49.616042+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:20.002373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"265'6657","reported_seq":7015,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.690437+0000","last_change":"2024-09-21T10:14:49.554445+0000","last_active":"2024-09-21T10:15:54.690437+0000","last_peered":"2024-09-21T10:15:54.690437+0000","last_clean":"2024-09-21T10:15:54.690437+0000","last_became_active":"2024-09-21T10:14:49.552633+0000","last_became_peered":"2024-09-21T10:14:49.552633+0000","last_unstale":"2024-09-21T10:15:54.690437+0000","last_undegraded":"2024-09-21T10:15:54.690437+0000","last_fullsized":"2024-09-21T10:15:54.690437+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6657,"log_dups_size":0,"ondisk_log_size":6657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:42.174531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436273152,"num_objects":6657,"num_object_clones":0,"num_object_copies":19971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6657,"num_write_kb":426048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":634,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115204+0000","last_change":"2024-09-21T10:13:12.573143+0000","last_active":"2024-09-21T10:15:36.115204+0000","last_peered":"2024-09-21T10:15:36.115204+0000","last_clean":"2024-09-21T10:15:36.115204+0000","last_became_active":"2024-09-21T10:13:12.571751+0000","last_became_peered":"2024-09-21T10:13:12.571751+0000","last_unstale":"2024-09-21T10:15:36.115204+0000","last_undegraded":"2024-09-21T10:15:36.115204+0000","last_fullsized":"2024-09-21T10:15:36.115204+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:28.804119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":583,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112591+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:15:36.112591+0000","last_peered":"2024-09-21T10:15:36.112591+0000","last_clean":"2024-09-21T10:15:36.112591+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:15:36.112591+0000","last_undegraded":"2024-09-21T10:15:36.112591+0000","last_fullsized":"2024-09-21T10:15:36.112591+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":352,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.602896+0000","last_change":"2024-09-21T10:14:45.527731+0000","last_active":"2024-09-21T10:15:49.602896+0000","last_peered":"2024-09-21T10:15:49.602896+0000","last_clean":"2024-09-21T10:15:49.602896+0000","last_became_active":"2024-09-21T10:14:45.527330+0000","last_became_peered":"2024-09-21T10:14:45.527330+0000","last_unstale":"2024-09-21T10:15:49.602896+0000","last_undegraded":"2024-09-21T10:15:49.602896+0000","last_fullsized":"2024-09-21T10:15:49.602896+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:03:48.411604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":218,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115873+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:15:36.115873+0000","last_peered":"2024-09-21T10:15:36.115873+0000","last_clean":"2024-09-21T10:15:36.115873+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:15:36.115873+0000","last_undegraded":"2024-09-21T10:15:36.115873+0000","last_fullsized":"2024-09-21T10:15:36.115873+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"265'6539","reported_seq":6568,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.667714+0000","last_change":"2024-09-21T10:14:28.440396+0000","last_active":"2024-09-21T10:15:54.667714+0000","last_peered":"2024-09-21T10:15:54.667714+0000","last_clean":"2024-09-21T10:15:54.667714+0000","last_became_active":"2024-09-21T10:14:28.440137+0000","last_became_peered":"2024-09-21T10:14:28.440137+0000","last_unstale":"2024-09-21T10:15:54.667714+0000","last_undegraded":"2024-09-21T10:15:54.667714+0000","last_fullsized":"2024-09-21T10:15:54.667714+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6539,"log_dups_size":0,"ondisk_log_size":6539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:33:10.456362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428539904,"num_objects":6539,"num_object_clones":0,"num_object_copies":19617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6539,"num_write_kb":418496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":858,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115168+0000","last_change":"2024-09-21T10:13:59.583227+0000","last_active":"2024-09-21T10:15:36.115168+0000","last_peered":"2024-09-21T10:15:36.115168+0000","last_clean":"2024-09-21T10:15:36.115168+0000","last_became_active":"2024-09-21T10:13:59.582623+0000","last_became_peered":"2024-09-21T10:13:59.582623+0000","last_unstale":"2024-09-21T10:15:36.115168+0000","last_undegraded":"2024-09-21T10:15:36.115168+0000","last_fullsized":"2024-09-21T10:15:36.115168+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:35.235896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":27,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118247+0000","last_change":"2024-09-21T10:14:32.461889+0000","last_active":"2024-09-21T10:15:36.118247+0000","last_peered":"2024-09-21T10:15:36.118247+0000","last_clean":"2024-09-21T10:15:36.118247+0000","last_became_active":"2024-09-21T10:14:32.461551+0000","last_became_peered":"2024-09-21T10:14:32.461551+0000","last_unstale":"2024-09-21T10:15:36.118247+0000","last_undegraded":"2024-09-21T10:15:36.118247+0000","last_fullsized":"2024-09-21T10:15:36.118247+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:30:45.261092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.604822+0000","last_change":"2024-09-21T10:13:09.580568+0000","last_active":"2024-09-21T10:15:49.604822+0000","last_peered":"2024-09-21T10:15:49.604822+0000","last_clean":"2024-09-21T10:15:49.604822+0000","last_became_active":"2024-09-21T10:13:09.579053+0000","last_became_peered":"2024-09-21T10:13:09.579053+0000","last_unstale":"2024-09-21T10:15:49.604822+0000","last_undegraded":"2024-09-21T10:15:49.604822+0000","last_fullsized":"2024-09-21T10:15:49.604822+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:39.206612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"265'6569","reported_seq":6596,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.586367+0000","last_change":"2024-09-21T10:14:32.642821+0000","last_active":"2024-09-21T10:15:54.586367+0000","last_peered":"2024-09-21T10:15:54.586367+0000","last_clean":"2024-09-21T10:15:54.586367+0000","last_became_active":"2024-09-21T10:14:32.642404+0000","last_became_peered":"2024-09-21T10:14:32.642404+0000","last_unstale":"2024-09-21T10:15:54.586367+0000","last_undegraded":"2024-09-21T10:15:54.586367+0000","last_fullsized":"2024-09-21T10:15:54.586367+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:02.051331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430505984,"num_objects":6569,"num_object_clones":0,"num_object_copies":19707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6569,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":921,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.601131+0000","last_change":"2024-09-21T10:13:59.590250+0000","last_active":"2024-09-21T10:15:49.601131+0000","last_peered":"2024-09-21T10:15:49.601131+0000","last_clean":"2024-09-21T10:15:49.601131+0000","last_became_active":"2024-09-21T10:13:59.587283+0000","last_became_peered":"2024-09-21T10:13:59.587283+0000","last_unstale":"2024-09-21T10:15:49.601131+0000","last_undegraded":"2024-09-21T10:15:49.601131+0000","last_fullsized":"2024-09-21T10:15:49.601131+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:14.118934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.604888+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:15:49.604888+0000","last_peered":"2024-09-21T10:15:49.604888+0000","last_clean":"2024-09-21T10:15:49.604888+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:15:49.604888+0000","last_undegraded":"2024-09-21T10:15:49.604888+0000","last_fullsized":"2024-09-21T10:15:49.604888+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.616121+0000","last_change":"2024-09-21T10:13:09.578776+0000","last_active":"2024-09-21T10:15:49.616121+0000","last_peered":"2024-09-21T10:15:49.616121+0000","last_clean":"2024-09-21T10:15:49.616121+0000","last_became_active":"2024-09-21T10:13:09.578462+0000","last_became_peered":"2024-09-21T10:13:09.578462+0000","last_unstale":"2024-09-21T10:15:49.616121+0000","last_undegraded":"2024-09-21T10:15:49.616121+0000","last_fullsized":"2024-09-21T10:15:49.616121+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:19.035767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"265'6563","reported_seq":6592,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.651563+0000","last_change":"2024-09-21T10:14:28.440111+0000","last_active":"2024-09-21T10:15:54.651563+0000","last_peered":"2024-09-21T10:15:54.651563+0000","last_clean":"2024-09-21T10:15:54.651563+0000","last_became_active":"2024-09-21T10:14:28.439733+0000","last_became_peered":"2024-09-21T10:14:28.439733+0000","last_unstale":"2024-09-21T10:15:54.651563+0000","last_undegraded":"2024-09-21T10:15:54.651563+0000","last_fullsized":"2024-09-21T10:15:54.651563+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6563,"log_dups_size":0,"ondisk_log_size":6563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:17.534207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430112768,"num_objects":6563,"num_object_clones":0,"num_object_copies":19689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6563,"num_write_kb":420032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1057,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112822+0000","last_change":"2024-09-21T10:13:12.560467+0000","last_active":"2024-09-21T10:15:36.112822+0000","last_peered":"2024-09-21T10:15:36.112822+0000","last_clean":"2024-09-21T10:15:36.112822+0000","last_became_active":"2024-09-21T10:13:12.559882+0000","last_became_peered":"2024-09-21T10:13:12.559882+0000","last_unstale":"2024-09-21T10:15:36.112822+0000","last_undegraded":"2024-09-21T10:15:36.112822+0000","last_fullsized":"2024-09-21T10:15:36.112822+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:33.267760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":137,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.601459+0000","last_change":"2024-09-21T10:13:56.570638+0000","last_active":"2024-09-21T10:15:49.601459+0000","last_peered":"2024-09-21T10:15:49.601459+0000","last_clean":"2024-09-21T10:15:49.601459+0000","last_became_active":"2024-09-21T10:13:56.570132+0000","last_became_peered":"2024-09-21T10:13:56.570132+0000","last_unstale":"2024-09-21T10:15:49.601459+0000","last_undegraded":"2024-09-21T10:15:49.601459+0000","last_fullsized":"2024-09-21T10:15:49.601459+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:28.451531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.605412+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:15:49.605412+0000","last_peered":"2024-09-21T10:15:49.605412+0000","last_clean":"2024-09-21T10:15:49.605412+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:15:49.605412+0000","last_undegraded":"2024-09-21T10:15:49.605412+0000","last_fullsized":"2024-09-21T10:15:49.605412+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"265'6486","reported_seq":6515,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.578692+0000","last_change":"2024-09-21T10:14:28.437169+0000","last_active":"2024-09-21T10:15:54.578692+0000","last_peered":"2024-09-21T10:15:54.578692+0000","last_clean":"2024-09-21T10:15:54.578692+0000","last_became_active":"2024-09-21T10:14:28.436873+0000","last_became_peered":"2024-09-21T10:14:28.436873+0000","last_unstale":"2024-09-21T10:15:54.578692+0000","last_undegraded":"2024-09-21T10:15:54.578692+0000","last_fullsized":"2024-09-21T10:15:54.578692+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6486,"log_dups_size":0,"ondisk_log_size":6486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:16.516429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425066496,"num_objects":6486,"num_object_clones":0,"num_object_copies":19458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6486,"num_write_kb":415104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":249,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.113026+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:15:36.113026+0000","last_peered":"2024-09-21T10:15:36.113026+0000","last_clean":"2024-09-21T10:15:36.113026+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:15:36.113026+0000","last_undegraded":"2024-09-21T10:15:36.113026+0000","last_fullsized":"2024-09-21T10:15:36.113026+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":688,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.605626+0000","last_change":"2024-09-21T10:13:59.577297+0000","last_active":"2024-09-21T10:15:49.605626+0000","last_peered":"2024-09-21T10:15:49.605626+0000","last_clean":"2024-09-21T10:15:49.605626+0000","last_became_active":"2024-09-21T10:13:59.576704+0000","last_became_peered":"2024-09-21T10:13:59.576704+0000","last_unstale":"2024-09-21T10:15:49.605626+0000","last_undegraded":"2024-09-21T10:15:49.605626+0000","last_fullsized":"2024-09-21T10:15:49.605626+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:11:28.773044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":125,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.614232+0000","last_change":"2024-09-21T10:14:45.531907+0000","last_active":"2024-09-21T10:15:49.614232+0000","last_peered":"2024-09-21T10:15:49.614232+0000","last_clean":"2024-09-21T10:15:49.614232+0000","last_became_active":"2024-09-21T10:14:45.531536+0000","last_became_peered":"2024-09-21T10:14:45.531536+0000","last_unstale":"2024-09-21T10:15:49.614232+0000","last_undegraded":"2024-09-21T10:15:49.614232+0000","last_fullsized":"2024-09-21T10:15:49.614232+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:18:52.155338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"265'6691","reported_seq":7025,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:54.710514+0000","last_change":"2024-09-21T10:14:48.563829+0000","last_active":"2024-09-21T10:15:54.710514+0000","last_peered":"2024-09-21T10:15:54.710514+0000","last_clean":"2024-09-21T10:15:54.710514+0000","last_became_active":"2024-09-21T10:14:48.549468+0000","last_became_peered":"2024-09-21T10:14:48.549468+0000","last_unstale":"2024-09-21T10:15:54.710514+0000","last_undegraded":"2024-09-21T10:15:54.710514+0000","last_fullsized":"2024-09-21T10:15:54.710514+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":245,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6691,"log_dups_size":0,"ondisk_log_size":6691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:08.229536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438501376,"num_objects":6691,"num_object_clones":0,"num_object_copies":20073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6691,"num_write_kb":428224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":118,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.605539+0000","last_change":"2024-09-21T10:14:45.530390+0000","last_active":"2024-09-21T10:15:49.605539+0000","last_peered":"2024-09-21T10:15:49.605539+0000","last_clean":"2024-09-21T10:15:49.605539+0000","last_became_active":"2024-09-21T10:14:45.530054+0000","last_became_peered":"2024-09-21T10:14:45.530054+0000","last_unstale":"2024-09-21T10:15:49.605539+0000","last_undegraded":"2024-09-21T10:15:49.605539+0000","last_fullsized":"2024-09-21T10:15:49.605539+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:42:49.775110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":746,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116613+0000","last_change":"2024-09-21T10:13:09.604050+0000","last_active":"2024-09-21T10:15:36.116613+0000","last_peered":"2024-09-21T10:15:36.116613+0000","last_clean":"2024-09-21T10:15:36.116613+0000","last_became_active":"2024-09-21T10:13:09.583593+0000","last_became_peered":"2024-09-21T10:13:09.583593+0000","last_unstale":"2024-09-21T10:15:36.116613+0000","last_undegraded":"2024-09-21T10:15:36.116613+0000","last_fullsized":"2024-09-21T10:15:36.116613+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:35.420105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":329,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112577+0000","last_change":"2024-09-21T10:14:45.528988+0000","last_active":"2024-09-21T10:15:36.112577+0000","last_peered":"2024-09-21T10:15:36.112577+0000","last_clean":"2024-09-21T10:15:36.112577+0000","last_became_active":"2024-09-21T10:14:45.528689+0000","last_became_peered":"2024-09-21T10:14:45.528689+0000","last_unstale":"2024-09-21T10:15:36.112577+0000","last_undegraded":"2024-09-21T10:15:36.112577+0000","last_fullsized":"2024-09-21T10:15:36.112577+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:48:43.539523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"265'6852","reported_seq":6902,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:57.654135+0000","last_change":"2024-09-21T10:14:28.438913+0000","last_active":"2024-09-21T10:15:57.654135+0000","last_peered":"2024-09-21T10:15:57.654135+0000","last_clean":"2024-09-21T10:15:57.654135+0000","last_became_active":"2024-09-21T10:14:28.438598+0000","last_became_peered":"2024-09-21T10:14:28.438598+0000","last_unstale":"2024-09-21T10:15:57.654135+0000","last_undegraded":"2024-09-21T10:15:57.654135+0000","last_fullsized":"2024-09-21T10:15:57.654135+0000","mapping_epoch":245,"log_start":"251'100","ondisk_log_start":"251'100","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:29:24.765631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":449052672,"num_objects":6852,"num_object_clones":0,"num_object_copies":20556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6852,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":399,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.604896+0000","last_change":"2024-09-21T10:13:56.647321+0000","last_active":"2024-09-21T10:15:49.604896+0000","last_peered":"2024-09-21T10:15:49.604896+0000","last_clean":"2024-09-21T10:15:49.604896+0000","last_became_active":"2024-09-21T10:13:56.644948+0000","last_became_peered":"2024-09-21T10:13:56.644948+0000","last_unstale":"2024-09-21T10:15:49.604896+0000","last_undegraded":"2024-09-21T10:15:49.604896+0000","last_fullsized":"2024-09-21T10:15:49.604896+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:40.992911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":715,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.605018+0000","last_change":"2024-09-21T10:13:12.569309+0000","last_active":"2024-09-21T10:15:49.605018+0000","last_peered":"2024-09-21T10:15:49.605018+0000","last_clean":"2024-09-21T10:15:49.605018+0000","last_became_active":"2024-09-21T10:13:12.568325+0000","last_became_peered":"2024-09-21T10:13:12.568325+0000","last_unstale":"2024-09-21T10:15:49.605018+0000","last_undegraded":"2024-09-21T10:15:49.605018+0000","last_fullsized":"2024-09-21T10:15:49.605018+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:18.314566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":407,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.601490+0000","last_change":"2024-09-21T10:13:09.579331+0000","last_active":"2024-09-21T10:15:49.601490+0000","last_peered":"2024-09-21T10:15:49.601490+0000","last_clean":"2024-09-21T10:15:49.601490+0000","last_became_active":"2024-09-21T10:13:09.578777+0000","last_became_peered":"2024-09-21T10:13:09.578777+0000","last_unstale":"2024-09-21T10:15:49.601490+0000","last_undegraded":"2024-09-21T10:15:49.601490+0000","last_fullsized":"2024-09-21T10:15:49.601490+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:50:34.425710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"265'6770","reported_seq":6821,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:57.651198+0000","last_change":"2024-09-21T10:14:28.439188+0000","last_active":"2024-09-21T10:15:57.651198+0000","last_peered":"2024-09-21T10:15:57.651198+0000","last_clean":"2024-09-21T10:15:57.651198+0000","last_became_active":"2024-09-21T10:14:28.438763+0000","last_became_peered":"2024-09-21T10:14:28.438763+0000","last_unstale":"2024-09-21T10:15:57.651198+0000","last_undegraded":"2024-09-21T10:15:57.651198+0000","last_fullsized":"2024-09-21T10:15:57.651198+0000","mapping_epoch":245,"log_start":"251'100","ondisk_log_start":"251'100","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6670,"log_dups_size":0,"ondisk_log_size":6670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:31.383586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443678720,"num_objects":6770,"num_object_clones":0,"num_object_copies":20310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6770,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":115,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.604448+0000","last_change":"2024-09-21T10:13:09.592519+0000","last_active":"2024-09-21T10:15:49.604448+0000","last_peered":"2024-09-21T10:15:49.604448+0000","last_clean":"2024-09-21T10:15:49.604448+0000","last_became_active":"2024-09-21T10:13:09.587615+0000","last_became_peered":"2024-09-21T10:13:09.587615+0000","last_unstale":"2024-09-21T10:15:49.604448+0000","last_undegraded":"2024-09-21T10:15:49.604448+0000","last_fullsized":"2024-09-21T10:15:49.604448+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:22:43.216612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":509,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116073+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:15:36.116073+0000","last_peered":"2024-09-21T10:15:36.116073+0000","last_clean":"2024-09-21T10:15:36.116073+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:15:36.116073+0000","last_undegraded":"2024-09-21T10:15:36.116073+0000","last_fullsized":"2024-09-21T10:15:36.116073+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":218,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118934+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:15:36.118934+0000","last_peered":"2024-09-21T10:15:36.118934+0000","last_clean":"2024-09-21T10:15:36.118934+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:15:36.118934+0000","last_undegraded":"2024-09-21T10:15:36.118934+0000","last_fullsized":"2024-09-21T10:15:36.118934+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":258,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118637+0000","last_change":"2024-09-21T10:15:21.038463+0000","last_active":"2024-09-21T10:15:36.118637+0000","last_peered":"2024-09-21T10:15:36.118637+0000","last_clean":"2024-09-21T10:15:36.118637+0000","last_became_active":"2024-09-21T10:15:21.038061+0000","last_became_peered":"2024-09-21T10:15:21.038061+0000","last_unstale":"2024-09-21T10:15:36.118637+0000","last_undegraded":"2024-09-21T10:15:36.118637+0000","last_fullsized":"2024-09-21T10:15:36.118637+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.241295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":240,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119477+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:15:36.119477+0000","last_peered":"2024-09-21T10:15:36.119477+0000","last_clean":"2024-09-21T10:15:36.119477+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:15:36.119477+0000","last_undegraded":"2024-09-21T10:15:36.119477+0000","last_fullsized":"2024-09-21T10:15:36.119477+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112564+0000","last_change":"2024-09-21T10:15:21.039385+0000","last_active":"2024-09-21T10:15:36.112564+0000","last_peered":"2024-09-21T10:15:36.112564+0000","last_clean":"2024-09-21T10:15:36.112564+0000","last_became_active":"2024-09-21T10:15:21.039073+0000","last_became_peered":"2024-09-21T10:15:21.039073+0000","last_unstale":"2024-09-21T10:15:36.112564+0000","last_undegraded":"2024-09-21T10:15:36.112564+0000","last_fullsized":"2024-09-21T10:15:36.112564+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:11.160836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":235,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115796+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:15:36.115796+0000","last_peered":"2024-09-21T10:15:36.115796+0000","last_clean":"2024-09-21T10:15:36.115796+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:15:36.115796+0000","last_undegraded":"2024-09-21T10:15:36.115796+0000","last_fullsized":"2024-09-21T10:15:36.115796+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":23,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112816+0000","last_change":"2024-09-21T10:14:45.528895+0000","last_active":"2024-09-21T10:15:36.112816+0000","last_peered":"2024-09-21T10:15:36.112816+0000","last_clean":"2024-09-21T10:15:36.112816+0000","last_became_active":"2024-09-21T10:14:45.528608+0000","last_became_peered":"2024-09-21T10:14:45.528608+0000","last_unstale":"2024-09-21T10:15:36.112816+0000","last_undegraded":"2024-09-21T10:15:36.112816+0000","last_fullsized":"2024-09-21T10:15:36.112816+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:48.887817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":338,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.601014+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:15:49.601014+0000","last_peered":"2024-09-21T10:15:49.601014+0000","last_clean":"2024-09-21T10:15:49.601014+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:15:49.601014+0000","last_undegraded":"2024-09-21T10:15:49.601014+0000","last_fullsized":"2024-09-21T10:15:49.601014+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":23,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112751+0000","last_change":"2024-09-21T10:14:45.530392+0000","last_active":"2024-09-21T10:15:36.112751+0000","last_peered":"2024-09-21T10:15:36.112751+0000","last_clean":"2024-09-21T10:15:36.112751+0000","last_became_active":"2024-09-21T10:14:45.530079+0000","last_became_peered":"2024-09-21T10:14:45.530079+0000","last_unstale":"2024-09-21T10:15:36.112751+0000","last_undegraded":"2024-09-21T10:15:36.112751+0000","last_fullsized":"2024-09-21T10:15:36.112751+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:17.306916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":417,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.601352+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:15:49.601352+0000","last_peered":"2024-09-21T10:15:49.601352+0000","last_clean":"2024-09-21T10:15:49.601352+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:15:49.601352+0000","last_undegraded":"2024-09-21T10:15:49.601352+0000","last_fullsized":"2024-09-21T10:15:49.601352+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":260,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.116026+0000","last_change":"2024-09-21T10:13:09.573620+0000","last_active":"2024-09-21T10:15:36.116026+0000","last_peered":"2024-09-21T10:15:36.116026+0000","last_clean":"2024-09-21T10:15:36.116026+0000","last_became_active":"2024-09-21T10:13:09.573216+0000","last_became_peered":"2024-09-21T10:13:09.573216+0000","last_unstale":"2024-09-21T10:15:36.116026+0000","last_undegraded":"2024-09-21T10:15:36.116026+0000","last_fullsized":"2024-09-21T10:15:36.116026+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:40.253863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":427,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.601592+0000","last_change":"2024-09-21T10:13:09.581932+0000","last_active":"2024-09-21T10:15:49.601592+0000","last_peered":"2024-09-21T10:15:49.601592+0000","last_clean":"2024-09-21T10:15:49.601592+0000","last_became_active":"2024-09-21T10:13:09.580798+0000","last_became_peered":"2024-09-21T10:13:09.580798+0000","last_unstale":"2024-09-21T10:15:49.601592+0000","last_undegraded":"2024-09-21T10:15:49.601592+0000","last_fullsized":"2024-09-21T10:15:49.601592+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:05.563826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":109,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.115442+0000","last_change":"2024-09-21T10:15:21.033453+0000","last_active":"2024-09-21T10:15:36.115442+0000","last_peered":"2024-09-21T10:15:36.115442+0000","last_clean":"2024-09-21T10:15:36.115442+0000","last_became_active":"2024-09-21T10:15:21.032893+0000","last_became_peered":"2024-09-21T10:15:21.032893+0000","last_unstale":"2024-09-21T10:15:36.115442+0000","last_undegraded":"2024-09-21T10:15:36.115442+0000","last_fullsized":"2024-09-21T10:15:36.115442+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:14.164715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":118,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.604545+0000","last_change":"2024-09-21T10:14:32.454735+0000","last_active":"2024-09-21T10:15:49.604545+0000","last_peered":"2024-09-21T10:15:49.604545+0000","last_clean":"2024-09-21T10:15:49.604545+0000","last_became_active":"2024-09-21T10:14:32.454475+0000","last_became_peered":"2024-09-21T10:14:32.454475+0000","last_unstale":"2024-09-21T10:15:49.604545+0000","last_undegraded":"2024-09-21T10:15:49.604545+0000","last_fullsized":"2024-09-21T10:15:49.604545+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:59.191649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":409,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.614484+0000","last_change":"2024-09-21T10:14:20.761163+0000","last_active":"2024-09-21T10:15:49.614484+0000","last_peered":"2024-09-21T10:15:49.614484+0000","last_clean":"2024-09-21T10:15:49.614484+0000","last_became_active":"2024-09-21T10:14:20.760814+0000","last_became_peered":"2024-09-21T10:14:20.760814+0000","last_unstale":"2024-09-21T10:15:49.614484+0000","last_undegraded":"2024-09-21T10:15:49.614484+0000","last_fullsized":"2024-09-21T10:15:49.614484+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:44.679743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":266,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119161+0000","last_change":"2024-09-21T10:13:09.577994+0000","last_active":"2024-09-21T10:15:36.119161+0000","last_peered":"2024-09-21T10:15:36.119161+0000","last_clean":"2024-09-21T10:15:36.119161+0000","last_became_active":"2024-09-21T10:13:09.577311+0000","last_became_peered":"2024-09-21T10:13:09.577311+0000","last_unstale":"2024-09-21T10:15:36.119161+0000","last_undegraded":"2024-09-21T10:15:36.119161+0000","last_fullsized":"2024-09-21T10:15:36.119161+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:27.532202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":423,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.601367+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:15:49.601367+0000","last_peered":"2024-09-21T10:15:49.601367+0000","last_clean":"2024-09-21T10:15:49.601367+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:15:49.601367+0000","last_undegraded":"2024-09-21T10:15:49.601367+0000","last_fullsized":"2024-09-21T10:15:49.601367+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":145,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118642+0000","last_change":"2024-09-21T10:13:09.578314+0000","last_active":"2024-09-21T10:15:36.118642+0000","last_peered":"2024-09-21T10:15:36.118642+0000","last_clean":"2024-09-21T10:15:36.118642+0000","last_became_active":"2024-09-21T10:13:09.577570+0000","last_became_peered":"2024-09-21T10:13:09.577570+0000","last_unstale":"2024-09-21T10:15:36.118642+0000","last_undegraded":"2024-09-21T10:15:36.118642+0000","last_fullsized":"2024-09-21T10:15:36.118642+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:21.299713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":111,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118541+0000","last_change":"2024-09-21T10:13:09.577841+0000","last_active":"2024-09-21T10:15:36.118541+0000","last_peered":"2024-09-21T10:15:36.118541+0000","last_clean":"2024-09-21T10:15:36.118541+0000","last_became_active":"2024-09-21T10:13:09.576793+0000","last_became_peered":"2024-09-21T10:13:09.576793+0000","last_unstale":"2024-09-21T10:15:36.118541+0000","last_undegraded":"2024-09-21T10:15:36.118541+0000","last_fullsized":"2024-09-21T10:15:36.118541+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:12.718520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":260,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112593+0000","last_change":"2024-09-21T10:13:09.589654+0000","last_active":"2024-09-21T10:15:36.112593+0000","last_peered":"2024-09-21T10:15:36.112593+0000","last_clean":"2024-09-21T10:15:36.112593+0000","last_became_active":"2024-09-21T10:13:09.574438+0000","last_became_peered":"2024-09-21T10:13:09.574438+0000","last_unstale":"2024-09-21T10:15:36.112593+0000","last_undegraded":"2024-09-21T10:15:36.112593+0000","last_fullsized":"2024-09-21T10:15:36.112593+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:12.504161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":27,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.118168+0000","last_change":"2024-09-21T10:14:32.771090+0000","last_active":"2024-09-21T10:15:36.118168+0000","last_peered":"2024-09-21T10:15:36.118168+0000","last_clean":"2024-09-21T10:15:36.118168+0000","last_became_active":"2024-09-21T10:14:32.770619+0000","last_became_peered":"2024-09-21T10:14:32.770619+0000","last_unstale":"2024-09-21T10:15:36.118168+0000","last_undegraded":"2024-09-21T10:15:36.118168+0000","last_fullsized":"2024-09-21T10:15:36.118168+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:29:31.473095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":219,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.113004+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:15:36.113004+0000","last_peered":"2024-09-21T10:15:36.113004+0000","last_clean":"2024-09-21T10:15:36.113004+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:15:36.113004+0000","last_undegraded":"2024-09-21T10:15:36.113004+0000","last_fullsized":"2024-09-21T10:15:36.113004+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":430,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.602478+0000","last_change":"2024-09-21T10:14:45.526997+0000","last_active":"2024-09-21T10:15:49.602478+0000","last_peered":"2024-09-21T10:15:49.602478+0000","last_clean":"2024-09-21T10:15:49.602478+0000","last_became_active":"2024-09-21T10:14:45.526600+0000","last_became_peered":"2024-09-21T10:14:45.526600+0000","last_unstale":"2024-09-21T10:15:49.602478+0000","last_undegraded":"2024-09-21T10:15:49.602478+0000","last_fullsized":"2024-09-21T10:15:49.602478+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:33:20.585859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":398,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-21T10:15:49.617184+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:15:49.617184+0000","last_peered":"2024-09-21T10:15:49.617184+0000","last_clean":"2024-09-21T10:15:49.617184+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:15:49.617184+0000","last_undegraded":"2024-09-21T10:15:49.617184+0000","last_fullsized":"2024-09-21T10:15:49.617184+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":249,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.119000+0000","last_change":"2024-09-21T10:13:09.577388+0000","last_active":"2024-09-21T10:15:36.119000+0000","last_peered":"2024-09-21T10:15:36.119000+0000","last_clean":"2024-09-21T10:15:36.119000+0000","last_became_active":"2024-09-21T10:13:09.577027+0000","last_became_peered":"2024-09-21T10:13:09.577027+0000","last_unstale":"2024-09-21T10:15:36.119000+0000","last_undegraded":"2024-09-21T10:15:36.119000+0000","last_fullsized":"2024-09-21T10:15:36.119000+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:49.522609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":29,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-21T10:15:36.112659+0000","last_change":"2024-09-21T10:14:45.536046+0000","last_active":"2024-09-21T10:15:36.112659+0000","last_peered":"2024-09-21T10:15:36.112659+0000","last_clean":"2024-09-21T10:15:36.112659+0000","last_became_active":"2024-09-21T10:14:45.530295+0000","last_became_peered":"2024-09-21T10:14:45.530295+0000","last_unstale":"2024-09-21T10:15:36.112659+0000","last_undegraded":"2024-09-21T10:15:36.112659+0000","last_fullsized":"2024-09-21T10:15:36.112659+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:34.920194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6961627136,"num_objects":106226,"num_object_clones":0,"num_object_copies":318678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":106226,"num_write_kb":6798464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20880424960,"data_stored":20880424960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104826,"ondisk_log_size":104826,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":1370,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":12582950,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234315,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2945064,"kb_used_data":2516896,"kb_used_omap":1,"kb_used_meta":428158,"kb_avail":90800088,"statfs":{"total":95995035648,"available":92979290112,"internally_reserved":0,"allocated":2577301504,"data_stored":2576051913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":438434251},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495953,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2894312,"kb_used_data":2498144,"kb_used_omap":1,"kb_used_meta":396158,"kb_avail":90850840,"statfs":{"total":95995035648,"available":93031260160,"internally_reserved":0,"allocated":2558099456,"data_stored":2556850651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":405666247},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757590,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2490456,"kb_used_data":2130384,"kb_used_omap":1,"kb_used_meta":360062,"kb_avail":91254696,"statfs":{"total":95995035648,"available":93444808704,"internally_reserved":0,"allocated":2181513216,"data_stored":2180275495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":368703942},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019226,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2906980,"kb_used_data":2532956,"kb_used_omap":1,"kb_used_meta":374014,"kb_avail":90838172,"statfs":{"total":95995035648,"available":93018288128,"internally_reserved":0,"allocated":2593746944,"data_stored":2592501166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":382990790},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435411,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5255248,"kb_used_data":4750152,"kb_used_omap":5,"kb_used_meta":505082,"kb_avail":88489904,"statfs":{"total":95995035648,"available":90613661696,"internally_reserved":0,"allocated":4864155648,"data_stored":4862925550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":517204008},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598937,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3011664,"kb_used_data":2510728,"kb_used_omap":7,"kb_used_meta":500920,"kb_avail":90733488,"statfs":{"total":95995035648,"available":92911091712,"internally_reserved":0,"allocated":2570985472,"data_stored":2569755592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":512942541},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762464,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3731860,"kb_used_data":3371148,"kb_used_omap":7,"kb_used_meta":360696,"kb_avail":90013292,"statfs":{"total":95995035648,"available":92173611008,"internally_reserved":0,"allocated":3452055552,"data_stored":3450821580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":369352957},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":403726925991,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3462856704,"data_stored":3462856704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2575237120,"data_stored":2575237120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4902551552,"data_stored":4902551552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2591752192,"data_stored":2591752192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2178613248,"data_stored":2178613248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2577661952,"data_stored":2577661952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2591752192,"data_stored":2591752192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:16:02.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 1 2024-09-21T10:16:02.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:02.775 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:02.806 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:02.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:02.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 79 16 113866 113850 90.057 66.5 0.0180143 0.0110987 2024-09-21T10:16:02.908 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:02.910 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:16:02.936 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:02.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:03.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:03.071 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:03.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:03.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:03.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:03.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:03.326 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:03.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:03.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:03.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:03.504 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:03.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:03.606 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:03.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.647 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:03.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:03.748 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:03.789 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:03.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:03.807 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:03 smithi032 ceph-mon[101299]: pgmap v770: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s 2024-09-21T10:16:03.807 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:03.807 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:03.807 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:03 smithi032 ceph-mon[105748]: pgmap v770: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s 2024-09-21T10:16:03.808 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:03.808 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:03.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:16:03.859686+0000 min lat: 0.00117819 max lat: 2.28911 avg lat: 0.0111414 2024-09-21T10:16:03.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:16:03.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 80 16 114868 114852 89.714 62.625 0.0132769 0.0111414 2024-09-21T10:16:03.891 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:03.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:03.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:04.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:03 smithi045 ceph-mon[95976]: pgmap v770: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s 2024-09-21T10:16:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:04.076 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:04.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:04.178 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:04.215 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:04.316 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:04.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.356 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:04.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:04.457 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:04.496 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:04.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:04.598 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:04.631 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:04.733 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:04.766 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:04.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:04.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 81 16 115880 115864 89.3872 63.25 0.0125705 0.0111823 2024-09-21T10:16:04.868 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:04.898 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:04.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:05.029 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:05.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.165 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:05.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:05.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:05.300 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:05.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:05.402 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:05.442 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:05.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:05.520 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:05.543 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:05.593 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:05.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.616 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:05.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:05.665 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:16:05.717 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:05.730 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:05 smithi032 ceph-mon[101299]: pgmap v771: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:05.730 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/756654918' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:05.730 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:05 smithi032 ceph-mon[105748]: pgmap v771: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:05.730 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/756654918' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:05.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.763 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:05.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:05.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 82 16 116830 116814 89.0212 59.375 0.00671852 0.0112278 2024-09-21T10:16:05.863 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:05.891 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:05.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:05.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:05 smithi045 ceph-mon[95976]: pgmap v771: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/756654918' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:06.038 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:06.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:06.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:06.171 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:06.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:06.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:06.314 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:06.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:06.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:06.454 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:06.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:06.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:06.586 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:06.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:06.688 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:06.742 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:06.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:06.844 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:06.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 83 16 117861 117845 88.7249 64.4375 0.0140816 0.0112657 2024-09-21T10:16:06.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:06 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/756654918' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:06 smithi032 ceph-mon[101299]: osdmap e266: 8 total, 8 up, 7 in 2024-09-21T10:16:06.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:06 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:06.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/756654918' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:06.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:06 smithi032 ceph-mon[105748]: osdmap e266: 8 total, 8 up, 7 in 2024-09-21T10:16:06.984 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:07.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:07.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:06 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/756654918' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:06 smithi045 ceph-mon[95976]: osdmap e266: 8 total, 8 up, 7 in 2024-09-21T10:16:07.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:07.159 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:07.261 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:07.299 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:07.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:07.401 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:07.407 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:07.407 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:07.437 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:07.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:07.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.585 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:07.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:07.686 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:07.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:07.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:07.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:07.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 84 16 118914 118898 88.4521 65.8125 0.0289836 0.0113005 2024-09-21T10:16:07.864 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:07.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:07 smithi032 ceph-mon[101299]: pgmap v773: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:16:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:07 smithi032 ceph-mon[105748]: pgmap v773: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:16:07.965 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:08.000 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:08.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:07 smithi045 ceph-mon[95976]: pgmap v773: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:16:08.102 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:08.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.143 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:08.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:08.244 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:08.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.284 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:08.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:08.385 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:08.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.424 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:08.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:08.524 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:08.562 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:08.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:08.663 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:08.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.760 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:08.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:08.861 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:08.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 85 15 119984 119969 88.199 66.9375 0.0212668 0.0113329 2024-09-21T10:16:08.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.908 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:08.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:09.009 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:09.031 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:16:09.042 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:09.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:09.143 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:09.200 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:09.303 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:09.349 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:09.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:09.450 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:09.485 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:09.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:09.587 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:09.625 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:09.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:09.727 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:09.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:09.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:09.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 86 16 120957 120941 87.8797 60.75 0.0106521 0.0113745 2024-09-21T10:16:09.870 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:09.900 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:09.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:09 smithi032 ceph-mon[105748]: pgmap v774: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:16:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:16:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:16:09] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.51.0" 2024-09-21T10:16:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:09 smithi032 ceph-mon[101299]: pgmap v774: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:16:10.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:09 smithi045 ceph-mon[95976]: pgmap v774: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:16:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:10.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:10.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:10.179 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:10.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:10.281 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:10.324 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:10.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:10.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:10.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:16:10.466 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:10.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:10.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:10.699 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.779 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:10.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:10.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 87 16 121924 121908 87.5643 60.4375 0.00965217 0.0114151 2024-09-21T10:16:10.881 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:10.922 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:10.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:11.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:11.066 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:11.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:11.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:11.211 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:11.312 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:11.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.372 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:11.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:11.473 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:11.509 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:11.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:11.652 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:11.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:11.754 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:11.794 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:11.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:11.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 88 16 122907 122891 87.2673 61.4375 0.00785262 0.0114532 2024-09-21T10:16:11.896 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:11.919 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:11.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:11 smithi032 ceph-mon[105748]: pgmap v775: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:11.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:11 smithi032 ceph-mon[101299]: pgmap v775: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:12.021 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:11 smithi045 ceph-mon[95976]: pgmap v775: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:12.057 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:12.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:12.158 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:12.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.200 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:12.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:12.301 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:12.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:12.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:12.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:12.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:12.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:12.573 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:12.611 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:12.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:12.712 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:12.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:12.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:12.861 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 89 16 123908 123892 86.9897 62.5625 0.00765764 0.0114908 2024-09-21T10:16:12.884 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:12.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:12.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:13.019 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:13.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:13.121 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:13.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:13.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:13.263 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:13.294 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:13.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:13.396 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:13.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.436 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:13.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:13.537 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:13.578 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:13.680 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:13.702 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:13.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:13.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:13.809 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:16:13.809 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:16:13.809 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:16:13.809 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:16:13.809 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:16:13.809 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:13.810 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:13.834 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:13.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total time run: 90.006 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total writes made: 124663 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Write size: 65536 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object size: 65536 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Bandwidth (MB/sec): 86.5657 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Bandwidth: 36.0657 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max bandwidth (MB/sec): 218.688 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average IOPS: 1385 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev IOPS: 577.051 2024-09-21T10:16:13.862 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max IOPS: 3499 2024-09-21T10:16:13.863 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min IOPS: 0 2024-09-21T10:16:13.863 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average Latency(s): 0.0115477 2024-09-21T10:16:13.863 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Latency(s): 0.0311233 2024-09-21T10:16:13.863 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max latency(s): 2.28911 2024-09-21T10:16:13.863 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min latency(s): 0.00117819 2024-09-21T10:16:13.866 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:16:13.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:13.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:13 smithi032 ceph-mon[101299]: pgmap v776: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:13.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1292905768' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:13.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:13 smithi032 ceph-mon[105748]: pgmap v776: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:13.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1292905768' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:13.971 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:13.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:13 smithi045 ceph-mon[95976]: pgmap v776: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1292905768' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:14.073 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:14.114 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:14.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:14.216 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.257 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:14.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:14.358 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:14.389 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:14.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:14.491 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:14.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:14.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:14.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:14.685 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:14.787 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:14.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:14.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:14.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.978 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:14.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:15.079 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:15.121 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:15.223 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:15.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.258 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:15.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:15.359 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:15.375 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:15.387 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:15.425 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:15.489 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:15.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.531 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:15.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:15.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:15.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:15.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:15.723 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 1 pgp_num to 1 2024-09-21T10:16:15.766 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:15.803 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:15.823 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:15 smithi032 ceph-mon[105748]: pgmap v777: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:15.823 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:15 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1659441825' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:16:15.824 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:15 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:16:15.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:15 smithi032 ceph-mon[101299]: pgmap v777: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:15.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:15 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1659441825' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:16:15.824 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:15 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:16:15.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:15.939 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:15.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:15 smithi045 ceph-mon[95976]: pgmap v777: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:16:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:15 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1659441825' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:16:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:15 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:16:16.040 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:16.084 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:16.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:16.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:16.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:16.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:16.324 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:16.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.361 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:16.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:16.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:16.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:16.603 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:16.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:16.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:16.742 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:16.776 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:16.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:16.878 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:16.903 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:16 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:16:16.903 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:16 smithi032 ceph-mon[101299]: osdmap e267: 8 total, 8 up, 7 in 2024-09-21T10:16:16.903 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:16 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:16:16.903 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:16 smithi032 ceph-mon[105748]: osdmap e267: 8 total, 8 up, 7 in 2024-09-21T10:16:16.908 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:17.010 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:16 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:16:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:16 smithi045 ceph-mon[95976]: osdmap e267: 8 total, 8 up, 7 in 2024-09-21T10:16:17.050 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:17.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:17.151 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:17.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.191 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:17.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:17.292 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:17.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:17.353 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:16:17.433 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:17.465 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:17.567 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:17.622 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:17.723 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:17.761 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:17.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.908 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:17.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:18.000 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:17 smithi032 ceph-mon[105748]: pgmap v779: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 58 MiB/s wr, 987 op/s 2024-09-21T10:16:18.000 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:17 smithi032 ceph-mon[101299]: pgmap v779: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 58 MiB/s wr, 987 op/s 2024-09-21T10:16:18.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:17 smithi045 ceph-mon[95976]: pgmap v779: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 58 MiB/s wr, 987 op/s 2024-09-21T10:16:18.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:18.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:18.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:18.192 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:18.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:18.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:18.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:18.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:18.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:18.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:18.618 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:18.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:18.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:18.758 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:18.845 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:18.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:18.891 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:18.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:18.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:19.028 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:19.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:19.113 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:19.113 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:19.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:19.187 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:19.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:19.289 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:19.334 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:19.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:19.436 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:19.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.477 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:19.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:19.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:19.602 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:19.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:19.704 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:19.733 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:19.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:19.745 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:16:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:16:19] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.51.0" 2024-09-21T10:16:19.822 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:16:19.822 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:16:19.822 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.822 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:130620 7.2 7:4010f8f9:::benchmark_data_smithi032_200324_object5954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.707426+0000", 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.032929238, 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.032958856000000002, 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 130620 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.823 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.707426+0000", 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.707426+0000", 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.707427+0000", 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.068e-06 2024-09-21T10:16:19.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.707431+0000", 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5850000000000001e-06 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.707442+0000", 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0454e-05 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.707449+0000", 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0140000000000001e-06 2024-09-21T10:16:19.825 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.721408+0000", 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013958981000000001 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.721549+0000", 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014082299999999999 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:19.826 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.721660+0000", 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000111611 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.725568+0000", 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0039076279999999998 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.738542+0000", 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012974197 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.827 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:130631 7.3 7:c71f4fe7:::benchmark_data_smithi032_200324_object5965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.714447+0000", 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.025908026000000001, 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.026014684999999999, 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 130631 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.828 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.714447+0000", 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.714447+0000", 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.714449+0000", 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.66e-06 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.829 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.714455+0000", 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.714468+0000", 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3485e-05 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.714477+0000", 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8429999999999999e-06 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.830 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.714545+0000", 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7822000000000002e-05 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.714663+0000", 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011796800000000001 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.714779+0000", 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000116372 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:19.831 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.724561+0000", 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0097820579999999997 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.724779+0000", 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00021741499999999999 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:130629 7.e e267/256 7:7f782ff0:::benchmark_data_smithi032_200324_object5963:head v 267'8227, mlcod=267'8224)", 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.722418+0000", 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.017937398, 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.01811343, 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.832 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.722418+0000", 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.722418+0000", 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.722418+0000", 2024-09-21T10:16:19.833 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8299999999999996e-07 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.722425+0000", 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1650000000000003e-06 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.722436+0000", 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1251e-05 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.722443+0000", 2024-09-21T10:16:19.834 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1910000000000003e-06 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.726113+0000", 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0036698550000000001 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.726135+0000", 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2268000000000001e-05 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.835 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:130635 7.a 7:53bda779:::benchmark_data_smithi032_200324_object5969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.725784+0000", 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.014570578000000001, 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.014796793000000001, 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 130635 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.836 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.725784+0000", 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.725784+0000", 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.725786+0000", 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.362e-06 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.725791+0000", 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.198e-06 2024-09-21T10:16:19.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.725809+0000", 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7921999999999999e-05 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.725818+0000", 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.02e-06 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.726504+0000", 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00068559199999999995 2024-09-21T10:16:19.838 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.726614+0000", 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011014999999999999 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.726727+0000", 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011283800000000001 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732684+0000", 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0059574520000000002 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.738555+0000", 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0058705809999999997 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:130638 7.a 7:597f5fcb:::benchmark_data_smithi032_200324_object5972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.727750+0000", 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.012605213000000001, 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012898473000000001, 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.840 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 130638 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727750+0000", 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727750+0000", 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.841 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727751+0000", 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0410000000000001e-06 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727756+0000", 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2599999999999996e-06 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727835+0000", 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9331000000000003e-05 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727845+0000", 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4369999999999996e-06 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732727+0000", 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0048816629999999996 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732849+0000", 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000122237 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:16:19.843 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732973+0000", 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012384000000000001 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:130636 7.6 e267/256 7:68e1488f:::benchmark_data_smithi032_200324_object5970:head v 267'8260, mlcod=267'8258)", 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.727907+0000", 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.012448489, 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012808702, 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:16:19.844 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727907+0000", 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727907+0000", 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727907+0000", 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7199999999999997e-07 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727912+0000", 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3640000000000001e-06 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727919+0000", 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5259999999999996e-06 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.727924+0000", 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7820000000000001e-06 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.846 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732443+0000", 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0045186169999999999 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732464+0000", 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1403000000000002e-05 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:130641 7.3 7:c4dd510d:::benchmark_data_smithi032_200324_object5975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-09-21T10:16:19.847 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.728119+0000", 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.012235888, 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012646671, 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 130641 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.728119+0000", 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.848 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.728119+0000", 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.728120+0000", 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6899999999999996e-07 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.728124+0000", 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8010000000000001e-06 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.849 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.728134+0000", 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0190000000000001e-05 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.728140+0000", 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.998e-06 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.728198+0000", 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7839e-05 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.850 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.728329+0000", 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013153700000000001 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.728461+0000", 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000131211 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732402+0000", 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0039413679999999998 2024-09-21T10:16:19.851 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.735496+0000", 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0030936129999999998 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:130634 7.e e267/256 7:793e8e9c:::benchmark_data_smithi032_200324_object5968:head v 267'8228, mlcod=267'8225)", 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.730367+0000", 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0099881880000000003, 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010464753, 2024-09-21T10:16:19.852 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.730367+0000", 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.730367+0000", 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.853 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.730368+0000", 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9599999999999998e-07 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.730375+0000", 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.99e-06 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.730385+0000", 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0823e-05 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.854 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.730393+0000", 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2110000000000001e-06 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.734597+0000", 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0042038789999999998 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.734616+0000", 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9060999999999999e-05 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.855 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:130642 7.a 7:5e8d4bb0:::benchmark_data_smithi032_200324_object5976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.732024+0000", 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0083310209999999992, 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0088554900000000006, 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 130642 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.856 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732024+0000", 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732024+0000", 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732026+0000", 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5430000000000001e-06 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732031+0000", 2024-09-21T10:16:19.857 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1420000000000001e-06 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732043+0000", 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2455e-05 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.732051+0000", 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.841e-06 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.738585+0000", 2024-09-21T10:16:19.858 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0065336539999999999 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.738718+0000", 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000133254 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.738834+0000", 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000115962 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.45206.0:130638 7.a e267/256)", 2024-09-21T10:16:19.859 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.737770+0000", 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0025847119999999999, 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0031628619999999998, 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.737770+0000", 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.737770+0000", 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.860 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.737771+0000", 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0210000000000001e-06 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.737781+0000", 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3009999999999995e-06 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.737792+0000", 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1586000000000001e-05 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.861 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.737799+0000", 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9190000000000002e-06 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:130618 7.2 7:42bc8d08:::benchmark_data_smithi032_200324_object5952:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.705308+0000", 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.035046893000000003, 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.035664616000000003, 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:19.862 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 130618 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.705308+0000", 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.705308+0000", 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.863 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.705310+0000", 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5650000000000001e-06 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.705316+0000", 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9869999999999996e-06 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.705329+0000", 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3123e-05 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.705343+0000", 2024-09-21T10:16:19.864 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4368e-05 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.710860+0000", 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0055168609999999996 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.710982+0000", 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000121958 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.711097+0000", 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000115036 2024-09-21T10:16:19.865 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.725564+0000", 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.014466611000000001 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.733260+0000", 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0076962319999999999 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:130644 7.f 7:feb4db3a:::benchmark_data_smithi032_200324_object5978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.739152+0000", 2024-09-21T10:16:19.866 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.001202584, 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0018863529999999999, 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 130644 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739152+0000", 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.867 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739152+0000", 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739154+0000", 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.279e-06 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739157+0000", 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.676e-06 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.868 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739168+0000", 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0822000000000001e-05 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739175+0000", 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0169999999999999e-06 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739295+0000", 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000120179 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739399+0000", 2024-09-21T10:16:19.869 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000103379 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739495+0000", 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6224999999999994e-05 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:130645 7.a 7.bde5305a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:16:19.739190+0000", 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.001165559, 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0019043409999999999, 2024-09-21T10:16:19.870 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 130645 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739190+0000", 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:19.871 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739190+0000", 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739190+0000", 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4900000000000001e-07 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739192+0000", 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0789999999999999e-06 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739198+0000", 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1820000000000003e-06 2024-09-21T10:16:19.872 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:16:19.739203+0000", 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.758e-06 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 13 2024-09-21T10:16:19.873 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:16:19.873 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:19.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.913 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:19.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:20.014 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:19 smithi045 ceph-mon[95976]: pgmap v780: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 52 MiB/s wr, 986 op/s 2024-09-21T10:16:20.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:20.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:20.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:20.188 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:20.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:19 smithi032 ceph-mon[105748]: pgmap v780: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 52 MiB/s wr, 986 op/s 2024-09-21T10:16:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:19 smithi032 ceph-mon[101299]: pgmap v780: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 52 MiB/s wr, 986 op/s 2024-09-21T10:16:20.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:20.335 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:20.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:20.437 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:20.466 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:20.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:20.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:20.603 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:20.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:20.705 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:20.742 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:20.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:20.844 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:20.878 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:20.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:20.979 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:21.010 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:21.112 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:21.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.146 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:21.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:21.247 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:21.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.289 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:21.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:21.390 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:21.430 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:21.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:21.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:21.569 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:21.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:21.671 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:21.707 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:21.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:21.809 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:21.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.847 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:21.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:21.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:21 smithi032 ceph-mon[101299]: pgmap v781: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 28 MiB/s wr, 988 op/s 2024-09-21T10:16:21.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:21 smithi032 ceph-mon[105748]: pgmap v781: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 28 MiB/s wr, 988 op/s 2024-09-21T10:16:21.948 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:21.977 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:21.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:21 smithi045 ceph-mon[95976]: pgmap v781: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 28 MiB/s wr, 988 op/s 2024-09-21T10:16:22.079 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:22.120 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:22.222 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:22.259 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:22.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:22.346 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:16:22.346 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:16:22.346 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-09-21T10:16:22.346 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd in 0 2024-09-21T10:16:22.361 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.430 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:22.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:22.452 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:16:22.531 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:22.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.626 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:22.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:22.727 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:22.761 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:22.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:22.792 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:16:22.800 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:22 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1551898453' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:16:22.800 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:22 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1551898453' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:16:22.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:22.902 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:22.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:23.003 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:22 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1551898453' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:16:23.043 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:23.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:23.145 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:23.188 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:23.290 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:23.330 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:23.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:23.432 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:23.465 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:23.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:23.567 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:23.605 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:23.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:23.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:23.748 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:23.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:23.879 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:23.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:23.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:23 smithi032 ceph-mon[105748]: pgmap v782: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 28 MiB/s wr, 988 op/s 2024-09-21T10:16:23.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:23 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:23.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1551898453' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:16:23.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:23 smithi032 ceph-mon[105748]: osdmap e268: 8 total, 8 up, 7 in 2024-09-21T10:16:23.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:23 smithi032 ceph-mon[101299]: pgmap v782: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 28 MiB/s wr, 988 op/s 2024-09-21T10:16:23.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:23 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:23.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1551898453' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:16:23.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:23 smithi032 ceph-mon[101299]: osdmap e268: 8 total, 8 up, 7 in 2024-09-21T10:16:23.974 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:23.981 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:16:23.981 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:16:23.981 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:24.011 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:24.013 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:24.018 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:16:24.018 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:23 smithi045 ceph-mon[95976]: pgmap v782: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 28 MiB/s wr, 988 op/s 2024-09-21T10:16:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:23 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1551898453' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:16:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:23 smithi045 ceph-mon[95976]: osdmap e268: 8 total, 8 up, 7 in 2024-09-21T10:16:24.051 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:16:24.051 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:24.090 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:16:24.091 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:16:24.113 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:24.124 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:16:24.124 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:24.144 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:24.164 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:16:24.164 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:24.197 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:16:24.197 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:24.230 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:16:24.230 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:24.245 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:24.275 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:24.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:24.377 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:24.412 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:24.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:24.537 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:24.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:24.638 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:24.678 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:24.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:24.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:24.816 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:24.918 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:24.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.958 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:24.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:24 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3931611780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:25.059 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.093 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:25.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:25.107 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:24 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3931611780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:25.107 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:24 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3931611780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:25.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:25.224 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:25.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:25.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:25.366 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:25.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:25.468 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:25.501 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:25.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:25.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:25.644 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:25.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:25.680 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:16:25.747 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:25.792 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:25.894 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:25.937 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:25.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:26.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:25 smithi045 ceph-mon[95976]: pgmap v784: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 1.5 MiB/s wr, 789 op/s 2024-09-21T10:16:26.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.072 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:26.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:26.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:26.200 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:25 smithi032 ceph-mon[101299]: pgmap v784: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 1.5 MiB/s wr, 789 op/s 2024-09-21T10:16:26.200 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:25 smithi032 ceph-mon[105748]: pgmap v784: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 1.5 MiB/s wr, 789 op/s 2024-09-21T10:16:26.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.212 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:26.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:26.313 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:26.357 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:26.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:26.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:26.498 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:26.600 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.638 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:26.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:26.740 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:26.788 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:26.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:26.890 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:26.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.929 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:26.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:27.030 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:27.064 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:27.166 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:27.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.197 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:27.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:27.298 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:27.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:27.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:27.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:27.470 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:27.572 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:27.630 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:27.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:27.731 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:27.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:27.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:27.867 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:27.895 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:27.901 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:27.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:27 smithi032 ceph-mon[101299]: pgmap v785: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 1.3 MiB/s wr, 968 op/s 2024-09-21T10:16:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:27 smithi032 ceph-mon[105748]: pgmap v785: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 1.3 MiB/s wr, 968 op/s 2024-09-21T10:16:28.003 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:28.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:28.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:28.138 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:28.171 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:28.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:28.279 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:27 smithi045 ceph-mon[95976]: pgmap v785: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 1.3 MiB/s wr, 968 op/s 2024-09-21T10:16:28.311 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:28.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:28.413 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:28.449 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:28.551 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:28.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.585 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:28.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:28.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:28.694 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:16:28.716 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:28.818 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:28.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.860 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:28.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:28.961 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:28.994 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:28.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:29.071 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:29.071 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:29.096 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:29.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:29.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:29.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:29.281 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:29.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:29.384 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:29.422 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:29.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:29.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:29.554 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:29.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:29.656 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:29.686 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:29.788 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:29.793 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:16:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:16:29] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.51.0" 2024-09-21T10:16:29.826 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:29.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:29.962 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:30.064 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:30.095 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:30.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:30.174 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:29 smithi032 ceph-mon[105748]: pgmap v786: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-21T10:16:30.174 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:29 smithi032 ceph-mon[101299]: pgmap v786: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-21T10:16:30.196 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:30.234 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:29 smithi045 ceph-mon[95976]: pgmap v786: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 969 op/s 2024-09-21T10:16:30.335 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:30.371 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:30.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:30.473 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:30.495 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:30.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:30.597 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:30.631 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:30.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:30.732 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:30.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.766 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:30.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:30.867 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:30.903 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:30.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:31.004 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:31.046 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:31.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:31.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:16:31.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:31.190 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:31.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:31.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:31.322 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:31.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:31.424 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:31.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:31.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:31.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:31.617 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:31.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:31.719 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:31.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.757 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:31.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:31.858 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:31.886 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:31.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:31.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:32.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:32.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:32.127 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:31 smithi045 ceph-mon[95976]: pgmap v787: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 964 op/s 2024-09-21T10:16:32.127 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:16:32.127 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:32.127 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:32.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:32.202 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:32.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:31 smithi032 ceph-mon[105748]: pgmap v787: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 964 op/s 2024-09-21T10:16:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:16:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:31 smithi032 ceph-mon[101299]: pgmap v787: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 964 op/s 2024-09-21T10:16:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:16:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:32.304 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:32.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:32.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.478 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:32.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:32.579 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:32.629 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:32.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:32.731 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:32.758 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:32.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:32.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:32.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:32.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:32.997 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:33.024 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:33.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:33.176 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:33.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:33.278 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:33.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:33.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:33.341 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:33.341 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:33.341 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:33 smithi032 ceph-mon[101299]: pgmap v788: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 964 op/s 2024-09-21T10:16:33.341 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:33.341 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:33.341 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:33 smithi032 ceph-mon[105748]: pgmap v788: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 964 op/s 2024-09-21T10:16:33.421 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:33.448 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:33.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:33 smithi045 ceph-mon[95976]: pgmap v788: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 964 op/s 2024-09-21T10:16:33.550 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:33.585 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:33.614 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:33 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:16:33.401+0000 7f532c355640 -1 mon.a@0(leader).osd e268 definitely_dead 0 2024-09-21T10:16:33.687 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:33.726 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:33.744 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:33.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:33.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:33.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:33.879 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:33.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:33.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:34.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.020 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:34.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:34.121 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:34.158 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:34.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:34.238 INFO:teuthology.orchestra.run.smithi032.stderr:marked in osd.0. 2024-09-21T10:16:34.250 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:16:34.259 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.307 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:34.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:34.408 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:34.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:34.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:34.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2493497873' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-21T10:16:34.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:34.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4101619919' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:34.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:34 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:34.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2493497873' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-21T10:16:34.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:34.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4101619919' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:34.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:34 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:34.532 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2493497873' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-21T10:16:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4101619919' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:34 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:34.554 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:34.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:34.656 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:34.678 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:34.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:34.801 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:34.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:34.903 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:34.925 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:35.026 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:35.048 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:35.150 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:35.173 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:35.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:35.274 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:35.296 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:35.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:35.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:35.421 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:35.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:16:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:16:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2493497873' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-21T10:16:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[101299]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T10:16:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[101299]: pgmap v790: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 956 op/s 2024-09-21T10:16:35.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:16:35.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:16:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2493497873' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-21T10:16:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[105748]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T10:16:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:35 smithi032 ceph-mon[105748]: pgmap v790: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 956 op/s 2024-09-21T10:16:35.522 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:35 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:16:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:35 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:16:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:35 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2493497873' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-21T10:16:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:35 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:35 smithi045 ceph-mon[95976]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T10:16:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:35 smithi045 ceph-mon[95976]: pgmap v790: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 956 op/s 2024-09-21T10:16:35.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:35.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:35.645 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:35.667 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:35.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:35.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:35.794 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:35.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:35.895 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:35.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:36.019 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:36.041 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:36.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:36.143 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:36.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.166 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:36.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:36.267 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:36.290 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:36.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:36.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:36.420 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:36.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:36 smithi032 ceph-mon[101299]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T10:16:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:36 smithi032 ceph-mon[105748]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T10:16:36.521 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:36 smithi045 ceph-mon[95976]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T10:16:36.562 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:36.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:36.664 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:36.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:36.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:36.818 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:36.855 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:36.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:36.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:36.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:36.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:37.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:37.138 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:37.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:37.240 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:37.278 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:37.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:37.380 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:37.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:37.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:37.516 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:37 smithi045 ceph-mon[95976]: 7.6 starting backfill to osd.0 from (0'0,0'0] MAX to 268'9132 2024-09-21T10:16:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:37 smithi045 ceph-mon[95976]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T10:16:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:37 smithi045 ceph-mon[95976]: 7.d starting backfill to osd.0 from (0'0,0'0] MAX to 268'9051 2024-09-21T10:16:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:37 smithi045 ceph-mon[95976]: 7.7 starting backfill to osd.0 from (0'0,0'0] MAX to 268'8984 2024-09-21T10:16:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:37 smithi045 ceph-mon[95976]: pgmap v793: 89 pgs: 5 peering, 84 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2024-09-21T10:16:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:37 smithi045 ceph-mon[95976]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T10:16:37.556 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:37.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:37.576 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[101299]: 7.6 starting backfill to osd.0 from (0'0,0'0] MAX to 268'9132 2024-09-21T10:16:37.576 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[101299]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[101299]: 7.d starting backfill to osd.0 from (0'0,0'0] MAX to 268'9051 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[101299]: 7.7 starting backfill to osd.0 from (0'0,0'0] MAX to 268'8984 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[101299]: pgmap v793: 89 pgs: 5 peering, 84 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[101299]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[105748]: 7.6 starting backfill to osd.0 from (0'0,0'0] MAX to 268'9132 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[105748]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[105748]: 7.d starting backfill to osd.0 from (0'0,0'0] MAX to 268'9051 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[105748]: 7.7 starting backfill to osd.0 from (0'0,0'0] MAX to 268'8984 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[105748]: pgmap v793: 89 pgs: 5 peering, 84 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2024-09-21T10:16:37.577 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:37 smithi032 ceph-mon[105748]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T10:16:37.658 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:37.689 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:37.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:37.769 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:16:37.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:37.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:37.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:37.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:38.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.025 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:38.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:38.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:38.156 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-09-21T10:16:38.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.191 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:38.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:38.292 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:38.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:38.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:38.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:38.475 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:38.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:38.577 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:38.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:38.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:38 smithi032 ceph-mon[101299]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T10:16:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:38.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:38.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:38 smithi032 ceph-mon[105748]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T10:16:38.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:38.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:38.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:38.743 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:38.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:38.844 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:38.898 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:38.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:38.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:39.032 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:39.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:38 smithi045 ceph-mon[95976]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T10:16:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:39.133 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:39.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:39.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.326 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:39.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:39.427 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:39.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.510 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:39.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:39.522 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:16:39.522 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:16:39.566 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:16:39.567 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:39.585 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:16:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:16:39] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.51.0" 2024-09-21T10:16:39.604 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:16:39.604 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:39.610 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:39.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.645 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:39.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:39.651 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:16:39.651 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:16:39.685 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:16:39.686 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:39.717 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:16:39.717 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:39.746 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:39.756 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:16:39.756 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:39.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.789 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:39.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:39.792 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:16:39.792 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:39.890 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:39.920 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:39.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:40.022 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:40.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.057 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:40.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:40.141 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:39 smithi032 ceph-mon[101299]: pgmap v796: 89 pgs: 5 peering, 84 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:16:40.141 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/358245244' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:40.141 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:39 smithi032 ceph-mon[105748]: pgmap v796: 89 pgs: 5 peering, 84 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:16:40.141 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/358245244' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:40.158 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:40.189 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:40.252 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:39 smithi045 ceph-mon[95976]: pgmap v796: 89 pgs: 5 peering, 84 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:16:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:39 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/358245244' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:40.291 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:40.317 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214062 5.11 5:8c4dff33:::benchmark_data_smithi032_140417_object80671:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883254+0000", 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43242649600001, 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1004434059999999, 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.321 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214062 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883254+0000", 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.322 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883254+0000", 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883255+0000", 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3300000000000003e-07 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883257+0000", 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0370000000000001e-06 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.323 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883263+0000", 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2949999999999999e-06 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883268+0000", 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0579999999999997e-06 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883732+0000", 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00046427600000000001 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.324 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883740+0000", 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1030000000000002e-06 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.964075+0000", 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0803345289999999 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.964178+0000", 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010293000000000001 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.325 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.964293+0000", 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011517499999999999 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.967088+0000", 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00279533 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.969883+0000", 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0027949390000000002 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.326 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.971970+0000", 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0020868950000000001 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.983516+0000", 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011545658 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.983570+0000", 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4113000000000002e-05 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.327 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.983697+0000", 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012763499999999999 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214089 5.11 5:8f54bdfd:::benchmark_data_smithi032_140417_object80698:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883291+0000", 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43238957099999, 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1006732299999999, 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.328 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214089 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883291+0000", 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883291+0000", 2024-09-21T10:16:40.329 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883291+0000", 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4900000000000001e-07 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883294+0000", 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1280000000000002e-06 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883300+0000", 2024-09-21T10:16:40.330 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6390000000000003e-06 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883307+0000", 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3799999999999999e-06 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883816+0000", 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00050982899999999995 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.331 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883824+0000", 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2529999999999998e-06 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.965643+0000", 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.081819053 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.965742+0000", 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.9646000000000001e-05 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.965868+0000", 2024-09-21T10:16:40.332 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012577899999999999 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.969899+0000", 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0040311610000000001 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.972021+0000", 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0021213600000000001 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.333 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.972123+0000", 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000102607 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.983802+0000", 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011678414999999999 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.983857+0000", 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5266999999999998e-05 2024-09-21T10:16:40.334 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.983964+0000", 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010726399999999999 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214132 5.11 5:89d5f004:::benchmark_data_smithi032_140417_object80741:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883324+0000", 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43235697799997, 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.100851228, 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.335 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214132 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883324+0000", 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883324+0000", 2024-09-21T10:16:40.336 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883324+0000", 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3599999999999999e-07 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883326+0000", 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0360000000000001e-06 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.337 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883332+0000", 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.28e-06 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883363+0000", 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0420999999999999e-05 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883863+0000", 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00050048099999999999 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.338 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883870+0000", 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8279999999999997e-06 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.967283+0000", 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0834128199999999 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.967399+0000", 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000116372 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.967488+0000", 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9128000000000005e-05 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.971989+0000", 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0045001449999999997 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.972072+0000", 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3121999999999995e-05 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.340 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975173+0000", 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0031016699999999999 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984002+0000", 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0088291529999999993 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984056+0000", 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3452999999999998e-05 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.341 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984175+0000", 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011888299999999999 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214137 5.11 5:8eca5c18:::benchmark_data_smithi032_140417_object80746:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883371+0000", 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43230963799999, 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1010141369999999, 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.342 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214137 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883371+0000", 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883371+0000", 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.343 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883371+0000", 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9100000000000004e-07 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883374+0000", 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2699999999999999e-06 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883380+0000", 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4910000000000002e-06 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883385+0000", 2024-09-21T10:16:40.344 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7280000000000003e-06 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883909+0000", 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00052426999999999997 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883917+0000", 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6529999999999994e-06 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.968423+0000", 2024-09-21T10:16:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0845063930000001 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.968536+0000", 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000113201 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.968686+0000", 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000149419 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.972087+0000", 2024-09-21T10:16:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0034009169999999998 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975241+0000", 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003154525 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975294+0000", 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2363e-05 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.347 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984212+0000", 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0089179710000000002 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984269+0000", 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7479000000000003e-05 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984385+0000", 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011596599999999999 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.348 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214143 5.11 5:8d0e7d88:::benchmark_data_smithi032_140417_object80752:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883391+0000", 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43228975800002, 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1012387720000001, 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214143 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883391+0000", 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883391+0000", 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883391+0000", 2024-09-21T10:16:40.350 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8000000000000001e-07 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883393+0000", 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.759e-06 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883399+0000", 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.66e-06 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.351 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883403+0000", 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7890000000000002e-06 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883956+0000", 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00055223200000000005 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883962+0000", 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9529999999999996e-06 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.352 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.969942+0000", 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.085980717 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.970053+0000", 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000110278 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.970171+0000", 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000118244 2024-09-21T10:16:40.353 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.972095+0000", 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001924582 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975332+0000", 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003236849 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.977514+0000", 2024-09-21T10:16:40.354 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0021820419999999999 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984439+0000", 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0069248790000000001 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984531+0000", 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1798000000000004e-05 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984630+0000", 2024-09-21T10:16:40.355 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8610000000000006e-05 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214150 5.11 5:89f8e177:::benchmark_data_smithi032_140417_object80759:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883409+0000", 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43227148099999, 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1014216729999999, 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.356 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214150 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883409+0000", 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883409+0000", 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.357 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883409+0000", 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1699999999999999e-07 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883411+0000", 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6559999999999999e-06 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883417+0000", 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4790000000000004e-06 2024-09-21T10:16:40.358 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883421+0000", 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.403e-06 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884011+0000", 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00058981500000000004 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884018+0000", 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4610000000000002e-06 2024-09-21T10:16:40.359 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.360 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.360 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.360 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.971004+0000", 2024-09-21T10:16:40.360 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0869860220000001 2024-09-21T10:16:40.360 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.360 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.360 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.360 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.971087+0000", 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2304000000000003e-05 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.971165+0000", 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8145999999999993e-05 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.972103+0000", 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00093817200000000003 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.361 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975391+0000", 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003287856 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.979159+0000", 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0037680090000000001 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984671+0000", 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0055119949999999996 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.362 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984719+0000", 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8699000000000002e-05 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984831+0000", 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000111339 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214223 5.11 5:89e63f95:::benchmark_data_smithi032_140417_object80832:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883427+0000", 2024-09-21T10:16:40.363 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.432254016, 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1017319000000001, 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214223 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883427+0000", 2024-09-21T10:16:40.364 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883427+0000", 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883427+0000", 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5699999999999998e-07 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883434+0000", 2024-09-21T10:16:40.365 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0779999999999998e-06 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883441+0000", 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0060000000000003e-06 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883447+0000", 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1449999999999996e-06 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.366 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884056+0000", 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00060897700000000002 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884063+0000", 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.72e-06 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.972162+0000", 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.088099645 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.367 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.972297+0000", 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013489099999999999 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.972451+0000", 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015360300000000001 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975189+0000", 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0027376950000000001 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.368 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975426+0000", 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000237629 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984870+0000", 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0094438479999999995 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985013+0000", 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014247000000000001 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.369 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985063+0000", 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9976999999999999e-05 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985158+0000", 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5859000000000005e-05 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214255 5.11 5:8b96d251:::benchmark_data_smithi032_140417_object80864:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.370 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883455+0000", 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43222543100001, 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.101918918, 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214255 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883455+0000", 2024-09-21T10:16:40.371 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883455+0000", 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883456+0000", 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7e-07 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883458+0000", 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6919999999999998e-06 2024-09-21T10:16:40.372 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883465+0000", 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5629999999999997e-06 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883470+0000", 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8450000000000002e-06 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884095+0000", 2024-09-21T10:16:40.373 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00062498800000000004 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884101+0000", 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2129999999999996e-06 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.973220+0000", 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0891194239999999 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.374 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.973305+0000", 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5038999999999997e-05 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.973387+0000", 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1748000000000004e-05 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975196+0000", 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0018088100000000001 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.375 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.978087+0000", 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0028913760000000002 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984906+0000", 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0068189119999999999 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985195+0000", 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00028914199999999999 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.376 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985244+0000", 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8816999999999997e-05 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985374+0000", 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012987900000000001 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214269 5.11 5:89d85e72:::benchmark_data_smithi032_140417_object80878:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883476+0000", 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43220464799998, 2024-09-21T10:16:40.377 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1021059900000001, 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214269 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883476+0000", 2024-09-21T10:16:40.378 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883476+0000", 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883476+0000", 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4799999999999999e-07 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883478+0000", 2024-09-21T10:16:40.379 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9010000000000001e-06 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883483+0000", 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.242e-06 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883487+0000", 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0280000000000002e-06 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884142+0000", 2024-09-21T10:16:40.380 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00065461700000000002 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884149+0000", 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.289e-06 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.974156+0000", 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0900071090000001 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.974284+0000", 2024-09-21T10:16:40.381 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000127222 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.974432+0000", 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014859399999999999 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975407+0000", 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00097492 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.382 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.980799+0000", 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0053920970000000002 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984942+0000", 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0041426889999999997 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985415+0000", 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00047291299999999999 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.383 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985467+0000", 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2071999999999999e-05 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985582+0000", 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114949 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214320 5.11 5:8bec5f85:::benchmark_data_smithi032_140417_object80929:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883493+0000", 2024-09-21T10:16:40.384 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43218796000002, 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.10228587, 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214320 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883493+0000", 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.385 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.386 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.386 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.386 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883493+0000", 2024-09-21T10:16:40.386 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.386 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.386 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.386 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883493+0000", 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.3000000000000002e-07 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883495+0000", 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.638e-06 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883500+0000", 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0329999999999999e-06 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.387 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883504+0000", 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6430000000000003e-06 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884196+0000", 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00069154500000000001 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884204+0000", 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9889999999999992e-06 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.388 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975467+0000", 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0912628040000001 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975591+0000", 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012463200000000001 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.975726+0000", 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013436000000000001 2024-09-21T10:16:40.389 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.977468+0000", 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001742546 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.980862+0000", 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0033938050000000002 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.984977+0000", 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0041150839999999998 2024-09-21T10:16:40.390 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985618+0000", 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00064097500000000001 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985667+0000", 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9218000000000002e-05 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985778+0000", 2024-09-21T10:16:40.391 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011126800000000001 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:214339 5.11 5:88245781:::benchmark_data_smithi032_140417_object80948:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:08:35.883509+0000", 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 484.43217109099999, 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.105752614, 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.392 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214339 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883509+0000", 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883509+0000", 2024-09-21T10:16:40.393 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883510+0000", 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0899999999999997e-07 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883512+0000", 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8950000000000001e-06 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.394 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883517+0000", 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9200000000000003e-06 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.883521+0000", 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9949999999999999e-06 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884245+0000", 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00072410200000000001 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.395 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:35.884252+0000", 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2289999999999997e-06 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.976453+0000", 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0922015220000001 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.976556+0000", 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000102167 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.396 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.976686+0000", 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0001304 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.977563+0000", 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00087683300000000004 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.981092+0000", 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0035289079999999999 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.397 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.985822+0000", 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.004730237 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.989004+0000", 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003182285 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.989116+0000", 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000111617 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.398 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:08:36.989262+0000", 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014619500000000001 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246087 6.0 6:0f91fc0c:::benchmark_data_smithi032_171826_object116464:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:55.968070+0000", 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 164.34761075, 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7894905400000001, 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.399 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246087 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968070+0000", 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968070+0000", 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968071+0000", 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3540000000000001e-06 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968078+0000", 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4620000000000001e-06 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.401 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968097+0000", 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9459000000000001e-05 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968107+0000", 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7089999999999997e-06 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972504+0000", 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0043967340000000002 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.402 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op must wait for map", 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972510+0000", 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7660000000000001e-06 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.980092+0000", 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0075817030000000004 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.980104+0000", 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1824000000000001e-05 2024-09-21T10:16:40.403 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.561582+0000", 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.581478614 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562038+0000", 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00045512100000000001 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.753782+0000", 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1917440509999999 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.404 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.757560+0000", 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0037787430000000002 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246089 6.0 6:0ab96db1:::benchmark_data_smithi032_171826_object116466:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:55.972378+0000", 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 164.34330281300001, 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.785609827, 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.405 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246089 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972378+0000", 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.406 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972378+0000", 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972379+0000", 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.113e-06 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972384+0000", 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2429999999999996e-06 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.407 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972395+0000", 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1054e-05 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972403+0000", 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1810000000000001e-06 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972560+0000", 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000156522 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.408 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972565+0000", 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8640000000000004e-06 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.980129+0000", 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.007564393 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.980137+0000", 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1990000000000006e-06 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.409 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562078+0000", 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.58194073199999996 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562089+0000", 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.08e-05 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.757913+0000", 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1958243159999999 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.410 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.757988+0000", 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4410000000000001e-05 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246161 6.0 6:0dcd9ebe:::benchmark_data_smithi032_171826_object116538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:56.067723+0000", 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 164.24795709099999, 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7490227869999999, 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.411 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246161 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067723+0000", 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067723+0000", 2024-09-21T10:16:40.412 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.413 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.413 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.413 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.413 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067724+0000", 2024-09-21T10:16:40.413 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.009e-06 2024-09-21T10:16:40.413 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.413 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.413 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.413 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067729+0000", 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7210000000000002e-06 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067740+0000", 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0417e-05 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067745+0000", 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4589999999999997e-06 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.414 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067798+0000", 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2703e-05 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067807+0000", 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0499999999999997e-06 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562194+0000", 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.49438728999999998 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.415 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562203+0000", 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6100000000000006e-06 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.758010+0000", 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1958076310000001 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.759077+0000", 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0010662219999999999 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.416 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1,2", 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.759646+0000", 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00056961100000000003 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798253+0000", 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.038607269 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798341+0000", 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7406999999999998e-05 2024-09-21T10:16:40.417 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.805652+0000", 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0073109749999999999 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816575+0000", 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010922967 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816634+0000", 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8971000000000002e-05 2024-09-21T10:16:40.418 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816746+0000", 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000112475 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246190 6.0 6:0bcfc80a:::benchmark_data_smithi032_171826_object116567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:56.680778+0000", 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 163.63490246000001, 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1366551, 2024-09-21T10:16:40.419 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246190 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680778+0000", 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.420 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680778+0000", 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680779+0000", 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.085e-06 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680784+0000", 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8350000000000003e-06 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.421 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680797+0000", 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2509e-05 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680802+0000", 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3720000000000004e-06 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680871+0000", 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9417000000000003e-05 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.422 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680883+0000", 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1939999999999999e-05 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.789771+0000", 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1088872789999999 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.789889+0000", 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000118028 2024-09-21T10:16:40.423 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1,2", 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.790005+0000", 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011647899999999999 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798294+0000", 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0082894289999999992 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798376+0000", 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1923999999999997e-05 2024-09-21T10:16:40.424 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.805752+0000", 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0073759380000000003 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.817252+0000", 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011500134 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.817307+0000", 2024-09-21T10:16:40.425 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4462999999999997e-05 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.817433+0000", 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012626799999999999 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246196 6.0 6:0d14539e:::benchmark_data_smithi032_171826_object116573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:56.690854+0000", 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 163.62482681700001, 2024-09-21T10:16:40.426 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1304781020000001, 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246196 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690854+0000", 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.427 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690854+0000", 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690855+0000", 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2160000000000001e-06 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690859+0000", 2024-09-21T10:16:40.428 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6200000000000001e-06 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690870+0000", 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1367e-05 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690877+0000", 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6660000000000002e-06 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.429 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690937+0000", 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0338999999999999e-05 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690948+0000", 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0713999999999999e-05 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798410+0000", 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1074626620000001 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.430 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798523+0000", 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0001131 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1,2", 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798656+0000", 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013238900000000001 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.805739+0000", 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0070834260000000003 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.431 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816458+0000", 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010718459 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816502+0000", 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4118e-05 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.821161+0000", 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0046593590000000001 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.432 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.821218+0000", 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7184e-05 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.821332+0000", 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000113483 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2387 7.a 7:5446da43:::benchmark_data_smithi032_200324_object2386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-09-21T10:16:40.433 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.514140+0000", 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 115.801540387, 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2841524579999999, 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2387 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.434 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514140+0000", 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514140+0000", 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514141+0000", 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3600000000000003e-07 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.435 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514164+0000", 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2730000000000001e-05 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514199+0000", 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5716000000000002e-05 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514207+0000", 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2810000000000003e-06 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.436 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514530+0000", 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00032313699999999998 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514676+0000", 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014658499999999999 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514823+0000", 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000146162 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.437 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.517811+0000", 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002988756 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.520195+0000", 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0023832739999999999 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.523380+0000", 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0031850680000000001 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.438 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.523389+0000", 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5000000000000005e-06 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521321+0000", 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.99793176100000003 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521346+0000", 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5525e-05 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.439 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.440 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798199+0000", 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.276853062 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798293+0000", 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3165000000000007e-05 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.441 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2395 7.a 7:58b2188b:::benchmark_data_smithi032_200324_object2394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.516536+0000", 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 115.799144301, 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2821265199999998, 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:40.442 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2395 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516536+0000", 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516536+0000", 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.443 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516537+0000", 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2799999999999995e-07 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516555+0000", 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8026e-05 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516565+0000", 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7599999999999997e-06 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.444 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516572+0000", 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1740000000000003e-06 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516761+0000", 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018932399999999999 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516886+0000", 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012498899999999999 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.517020+0000", 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013357500000000001 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518943+0000", 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001922813 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.524409+0000", 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0054666100000000002 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.446 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.524420+0000", 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1077000000000001e-05 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521389+0000", 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.99696905700000005 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521401+0000", 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1167e-05 2024-09-21T10:16:40.447 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798572+0000", 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2771710330000001 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798663+0000", 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1187000000000006e-05 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.448 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2437 7.a 7:5ed8f833:::benchmark_data_smithi032_200324_object2436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.546186+0000", 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 115.769494505, 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2556801950000001, 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2437 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.449 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546186+0000", 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546186+0000", 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546187+0000", 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0299999999999998e-07 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.450 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546238+0000", 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1282999999999997e-05 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546262+0000", 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4000000000000001e-05 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546270+0000", 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0649999999999999e-06 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.451 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546345+0000", 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5184000000000004e-05 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546357+0000", 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1989e-05 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521431+0000", 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.97507342799999996 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.452 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521439+0000", 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3159999999999997e-06 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798712+0000", 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2772729140000001 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798852+0000", 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000139624 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.453 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799003+0000", 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015124999999999999 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800437+0000", 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014343559999999999 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801083+0000", 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00064619600000000003 2024-09-21T10:16:40.454 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801513+0000", 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00042902100000000002 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801585+0000", 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2937999999999999e-05 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801866+0000", 2024-09-21T10:16:40.455 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000280828 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2439 7.a 7:5a372019:::benchmark_data_smithi032_200324_object2438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.548988+0000", 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 115.766692658, 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2546032619999998, 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:16:40.456 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2439 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.548988+0000", 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.548988+0000", 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.457 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.548990+0000", 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1490000000000001e-06 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549047+0000", 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6978000000000002e-05 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549066+0000", 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8821000000000002e-05 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.458 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549075+0000", 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4080000000000004e-06 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549139+0000", 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3781000000000005e-05 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549147+0000", 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3049999999999993e-06 2024-09-21T10:16:40.459 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521467+0000", 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.97231946499999999 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521475+0000", 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.678e-06 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799623+0000", 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2781484750000001 2024-09-21T10:16:40.460 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799792+0000", 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016894100000000001 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799965+0000", 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017336700000000001 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801098+0000", 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001132463 2024-09-21T10:16:40.461 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801924+0000", 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00082578700000000003 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.802129+0000", 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000205732 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.802360+0000", 2024-09-21T10:16:40.462 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00023111599999999999 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.803591+0000", 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0012307959999999999 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:16:40.463 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:16:40.470 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:40.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:40.572 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:40.601 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:40.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:40.703 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:40.762 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:40.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:40.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:40.911 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:40.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:41.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:41.058 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:41.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:41.160 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:41.187 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:41.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:41.289 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:41.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.330 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:41.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:41.431 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:41.473 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:41.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:41.576 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:41.604 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:41.706 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:41.742 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:41.844 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:41.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:41 smithi032 ceph-mon[105748]: pgmap v797: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 18584/307608 objects misplaced (6.041%); 6.9 MiB/s, 104 objects/s recovering 2024-09-21T10:16:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:41 smithi032 ceph-mon[101299]: pgmap v797: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 18584/307608 objects misplaced (6.041%); 6.9 MiB/s, 104 objects/s recovering 2024-09-21T10:16:41.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:42.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:42.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:42.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:42.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:42.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:42.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:41 smithi045 ceph-mon[95976]: pgmap v797: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 18584/307608 objects misplaced (6.041%); 6.9 MiB/s, 104 objects/s recovering 2024-09-21T10:16:42.317 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:42.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:42.419 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:42.449 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:42.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:42.550 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:42.595 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:42.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:42.697 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:42.727 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:42.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:42.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:42.865 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:42.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:42.966 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:43.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.005 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:43.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:43.106 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:43.138 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:43.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:43.157 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:16:43.157 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:16:43.157 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-21T10:16:43.157 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 1.000000 2024-09-21T10:16:43.158 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 4 1 2024-09-21T10:16:43.239 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:43.270 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:43.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:43.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:43.423 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:43.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:43.479 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:16:43.525 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:43.580 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:43.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:43.682 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:43.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:43.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:43.811 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:16:43.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:43.864 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:43.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:43.966 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:44.004 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:44.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:44.057 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:44.105 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:44.150 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:43 smithi045 ceph-mon[95976]: pgmap v798: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 18584/307608 objects misplaced (6.041%); 5.8 MiB/s, 87 objects/s recovering 2024-09-21T10:16:44.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.179 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:44.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:43 smithi032 ceph-mon[105748]: pgmap v798: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 18584/307608 objects misplaced (6.041%); 5.8 MiB/s, 87 objects/s recovering 2024-09-21T10:16:44.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:43 smithi032 ceph-mon[101299]: pgmap v798: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 18584/307608 objects misplaced (6.041%); 5.8 MiB/s, 87 objects/s recovering 2024-09-21T10:16:44.280 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:44.319 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:44.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:44.421 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:44.461 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:44.563 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:44.607 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:44.709 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:44.738 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:44.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:44.771 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:16:44.839 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:44.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.910 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:44.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:45.011 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:45.037 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:45.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:45.139 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:45.185 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:45.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:45.287 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:45.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:45.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:45.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:45.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:45.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:45.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:45.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:45.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:45.700 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:45.747 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:45.849 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:45.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:45.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:45.892 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:45 smithi032 ceph-mon[105748]: pgmap v799: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 18584/302847 objects misplaced (6.136%); 4.4 MiB/s, 67 objects/s recovering 2024-09-21T10:16:45.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.032 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:46.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:45 smithi045 ceph-mon[95976]: pgmap v799: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 18584/302847 objects misplaced (6.136%); 4.4 MiB/s, 67 objects/s recovering 2024-09-21T10:16:46.080 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:16:46.080 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:16:46.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.080 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:152381 7.1 7:8dc7413e:::benchmark_data_smithi032_200324_object27715:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e273)", 2024-09-21T10:16:46.080 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:16:46.054023+0000", 2024-09-21T10:16:46.080 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.021848632999999999, 2024-09-21T10:16:46.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.021879968999999999, 2024-09-21T10:16:46.080 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 152381 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.054023+0000", 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.054023+0000", 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.054021+0000", 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.054029+0000", 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4250000000000001e-06 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.054049+0000", 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9975000000000001e-05 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.054057+0000", 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4589999999999998e-06 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.058173+0000", 2024-09-21T10:16:46.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0041159739999999997 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.058289+0000", 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011568 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.058409+0000", 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120569 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.066997+0000", 2024-09-21T10:16:46.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0085879430000000007 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.067454+0000", 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00045675499999999999 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:152389 7.9 e273/248 7:953a8ecf:::benchmark_data_smithi032_200324_object27723:head v 273'9513, mlcod=273'9510)", 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:16:46.068136+0000", 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0077354629999999997, 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0078533319999999993, 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:46.085 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068136+0000", 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068136+0000", 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:46.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068135+0000", 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068143+0000", 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7170000000000008e-06 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068158+0000", 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5478999999999999e-05 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068167+0000", 2024-09-21T10:16:46.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1219999999999995e-06 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068351+0000", 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00018484799999999999 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068428+0000", 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6680000000000004e-05 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:152392 7.4 7:27b4e3df:::benchmark_data_smithi032_200324_object27726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e273)", 2024-09-21T10:16:46.088 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:16:46.068773+0000", 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0070984530000000002, 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0072740189999999996, 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 152392 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068773+0000", 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:46.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068773+0000", 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068772+0000", 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068778+0000", 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.268e-06 2024-09-21T10:16:46.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068792+0000", 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4185999999999999e-05 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068799+0000", 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1899999999999998e-06 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.068935+0000", 2024-09-21T10:16:46.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013604900000000001 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.069255+0000", 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00031922400000000001 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,6", 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.069692+0000", 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00043710899999999998 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.074859+0000", 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0051675979999999998 2024-09-21T10:16:46.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.074895+0000", 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5221e-05 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:152393 7.9 e273/248 7:9d862eeb:::benchmark_data_smithi032_200324_object27727:head v 273'9514, mlcod=273'9510)", 2024-09-21T10:16:46.093 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:16:46.070740+0000", 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0051321689999999998, 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0053864550000000001, 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.070740+0000", 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.070740+0000", 2024-09-21T10:16:46.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.070739+0000", 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.070746+0000", 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1269999999999997e-06 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:46.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.070759+0000", 2024-09-21T10:16:46.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3298999999999999e-05 2024-09-21T10:16:46.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.070766+0000", 2024-09-21T10:16:46.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9840000000000004e-06 2024-09-21T10:16:46.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:46.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.075705+0000", 2024-09-21T10:16:46.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049392790000000004 2024-09-21T10:16:46.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:46.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.075724+0000", 2024-09-21T10:16:46.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8983000000000001e-05 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:152396 7.9 e273/248)", 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:16:46.074617+0000", 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0012550650000000001, 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015596679999999999, 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:46.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.074617+0000", 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.074617+0000", 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.074616+0000", 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.074623+0000", 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6669999999999999e-06 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.074636+0000", 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2763e-05 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.074642+0000", 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2920000000000001e-06 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:152378 7.1 7:838e925d:::benchmark_data_smithi032_200324_object27712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e273)", 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:16:46.049397+0000", 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.026474694, 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.026826244999999999, 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:46.101 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 152378 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.049397+0000", 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.049397+0000", 2024-09-21T10:16:46.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.049395+0000", 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999976 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.049403+0000", 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0620000000000001e-06 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.049418+0000", 2024-09-21T10:16:46.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4952e-05 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.049426+0000", 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4260000000000003e-06 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.051090+0000", 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001663311 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.051433+0000", 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00034366799999999999 2024-09-21T10:16:46.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.052053+0000", 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00062010600000000004 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.062587+0000", 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010533189 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:46.066815+0000", 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0042282320000000002 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.105 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:46.106 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.106 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:46.106 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:16:46.106 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 6 2024-09-21T10:16:46.106 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:16:46.132 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.182 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:46.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:45 smithi032 ceph-mon[101299]: pgmap v799: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 18584/302847 objects misplaced (6.136%); 4.4 MiB/s, 67 objects/s recovering 2024-09-21T10:16:46.283 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:46.312 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:46.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:46.413 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:46.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:46.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:46.692 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.746 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:46.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:46.846 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:46.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:46.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:46.898 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:16:46.905 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:16:46.998 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:47.038 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:47.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:47.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:47.189 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:47.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:46 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2471863329' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:46 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:46 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2471863329' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:46 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:47.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:46 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2471863329' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:47.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:46 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:47.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:47.310 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:16:47.326 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:47.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:47.428 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:47.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:47.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:47.582 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.630 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:47.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:47.730 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:47.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:47.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:47.878 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:47.915 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:47.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:48.016 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:48.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:47 smithi045 ceph-mon[95976]: pgmap v800: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 17030/295413 objects misplaced (5.765%); 9.0 MiB/s, 140 objects/s recovering 2024-09-21T10:16:48.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:47 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:48.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:47 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:48.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:47 smithi045 ceph-mon[95976]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T10:16:48.059 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:48.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:48.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:47 smithi032 ceph-mon[101299]: pgmap v800: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 17030/295413 objects misplaced (5.765%); 9.0 MiB/s, 140 objects/s recovering 2024-09-21T10:16:48.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:47 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:48.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:47 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:48.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:47 smithi032 ceph-mon[101299]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T10:16:48.149 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:47 smithi032 ceph-mon[105748]: pgmap v800: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 17030/295413 objects misplaced (5.765%); 9.0 MiB/s, 140 objects/s recovering 2024-09-21T10:16:48.149 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:47 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:48.149 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:47 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:48.149 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:47 smithi032 ceph-mon[105748]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T10:16:48.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:48.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:48.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:48.303 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:48.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.341 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:48.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:48.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:48.474 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:48.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:48.575 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:48.611 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:48.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:48.670 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:48.712 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:48.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:48.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:48.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:48.900 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:48.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:49.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:49.056 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:49.056 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:49.057 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:49.099 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:49.158 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:49.192 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:49.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:49.264 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:16:49.265 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:16:49.265 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:16:49.265 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:16:49.265 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:16:49.293 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:49.335 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:49.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:49.436 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:49.492 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:49.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:49.594 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:49.623 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:49.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:49.688 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:16:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:16:49] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.51.0" 2024-09-21T10:16:49.725 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:49.768 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:49.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:49.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.918 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:49.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:49.949 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:16:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:49 smithi032 ceph-mon[105748]: pgmap v802: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 17030/292455 objects misplaced (5.823%); 8.8 MiB/s, 137 objects/s recovering 2024-09-21T10:16:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:49 smithi032 ceph-mon[101299]: pgmap v802: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 17030/292455 objects misplaced (5.823%); 8.8 MiB/s, 137 objects/s recovering 2024-09-21T10:16:50.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:50.055 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:50.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:16:50.157 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:50.227 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:50.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:49 smithi045 ceph-mon[95976]: pgmap v802: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 17030/292455 objects misplaced (5.823%); 8.8 MiB/s, 137 objects/s recovering 2024-09-21T10:16:50.293 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:16:50.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:50.370 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:50.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:50.472 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:50.523 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:50.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:50.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:50.671 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:50.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:50.773 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:50.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:50.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:50.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:50.916 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-21T10:16:50.974 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:51.076 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:51.125 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:51.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:50 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2049519792' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:16:51.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:50 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:16:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:50 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2049519792' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:16:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:50 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:16:51.228 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:51.270 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:51.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:51.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:50 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2049519792' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:16:51.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:50 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:16:51.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:51.439 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:51.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:51.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:51.587 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:51.689 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:51.716 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:51.818 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:51.864 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:51.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:51.966 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:52.009 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:52.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:52.146 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:52.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:52.153 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:51 smithi045 ceph-mon[95976]: pgmap v803: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 15641/283380 objects misplaced (5.519%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:16:52.153 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:51 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:16:52.153 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:51 smithi045 ceph-mon[95976]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T10:16:52.153 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:52.153 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[105748]: pgmap v803: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 15641/283380 objects misplaced (5.519%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:16:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:16:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[105748]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T10:16:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[101299]: pgmap v803: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 15641/283380 objects misplaced (5.519%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:16:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:16:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[101299]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T10:16:52.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:52.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:207624 5.11 5:8efdf192:::benchmark_data_smithi032_140417_object74233:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:27.042085+0000", 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 505.16565059800001, 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.87776399199999999, 2024-09-21T10:16:52.213 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 207624 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.042085+0000", 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.214 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.042085+0000", 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.042082+0000", 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999976 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.042093+0000", 2024-09-21T10:16:52.215 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0621999999999999e-05 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.042121+0000", 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7959000000000001e-05 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.042131+0000", 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0359e-05 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.216 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.163773+0000", 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.121642245 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.163784+0000", 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0934999999999999e-05 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.917352+0000", 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.75356798700000005 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:16:52.217 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.917487+0000", 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013491999999999999 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.919775+0000", 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022877430000000001 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:27.919849+0000", 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3782999999999995e-05 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.218 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226274 6.e 6:746225e3:::benchmark_data_smithi032_171826_object96651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.671448+0000", 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.536287058, 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.008614471, 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226274 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.219 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.671448+0000", 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.671448+0000", 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.671446+0000", 2024-09-21T10:16:52.220 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.671456+0000", 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5899999999999997e-06 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.671472+0000", 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6206e-05 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.671482+0000", 2024-09-21T10:16:52.221 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8139999999999995e-06 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.674918+0000", 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0034359210000000002 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.674929+0000", 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0798999999999999e-05 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.222 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.667663+0000", 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99273434699999996 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.667803+0000", 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014011199999999999 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.667951+0000", 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014834000000000001 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.223 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.671654+0000", 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003703039 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.671660+0000", 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7660000000000004e-06 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.679969+0000", 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0083086470000000006 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.224 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680020+0000", 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1217000000000001e-05 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680063+0000", 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2585e-05 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.225 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226280 6.e 6:7725c644:::benchmark_data_smithi032_171826_object96657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.679082+0000", 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.52865313199999, 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.001241168, 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226280 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.226 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.227 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.679082+0000", 2024-09-21T10:16:52.227 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.227 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.227 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.227 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.227 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.679082+0000", 2024-09-21T10:16:52.227 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.227 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.227 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.679080+0000", 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.679088+0000", 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0439999999999996e-06 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.679106+0000", 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7844000000000001e-05 2024-09-21T10:16:52.228 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.679114+0000", 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5620000000000004e-06 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.679276+0000", 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016135999999999999 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.679329+0000", 2024-09-21T10:16:52.229 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2880999999999997e-05 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.671705+0000", 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99237645600000002 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.671829+0000", 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012349400000000001 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.230 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.671963+0000", 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000134299 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.679917+0000", 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0079542750000000002 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680145+0000", 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000227901 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.231 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680247+0000", 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000101968 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680284+0000", 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6897000000000003e-05 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680323+0000", 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9190999999999998e-05 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.232 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226283 6.e 6:7a583f40:::benchmark_data_smithi032_171826_object96660:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.681423+0000", 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.526311873, 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99902484499999999, 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226283 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.233 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.681423+0000", 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.681423+0000", 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.681422+0000", 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.234 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.681430+0000", 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1580000000000005e-06 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.681449+0000", 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8902e-05 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.681485+0000", 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5877000000000002e-05 2024-09-21T10:16:52.235 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.681537+0000", 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2401000000000002e-05 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.681548+0000", 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0859000000000001e-05 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.675074+0000", 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99352580999999995 2024-09-21T10:16:52.236 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.675223+0000", 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014890199999999999 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.675387+0000", 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016419399999999999 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680195+0000", 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0048083620000000001 2024-09-21T10:16:52.237 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680332+0000", 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000137107 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680377+0000", 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4481000000000003e-05 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680427+0000", 2024-09-21T10:16:52.238 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0114000000000002e-05 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680448+0000", 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1339999999999999e-05 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226314 6.e 6:7b6080a4:::benchmark_data_smithi032_171826_object96691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.716680+0000", 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.49105556699999, 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97638483700000001, 2024-09-21T10:16:52.239 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226314 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.716680+0000", 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.240 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.716680+0000", 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.716678+0000", 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.716686+0000", 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6340000000000007e-06 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.241 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.716706+0000", 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9639999999999998e-05 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.716715+0000", 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8449999999999992e-06 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.716773+0000", 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8795000000000002e-05 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.242 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.716787+0000", 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3346000000000001e-05 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680479+0000", 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96369226699999999 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680562+0000", 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3035999999999996e-05 2024-09-21T10:16:52.243 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.680657+0000", 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5340000000000005e-05 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.692806+0000", 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012148874 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.692901+0000", 2024-09-21T10:16:52.244 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4457999999999996e-05 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.692988+0000", 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6957000000000001e-05 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693030+0000", 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1961e-05 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.245 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693064+0000", 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4739000000000001e-05 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226319 6.e 6:71a4b838:::benchmark_data_smithi032_171826_object96696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.719349+0000", 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.48838593599999, 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97382549799999996, 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.246 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226319 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.719349+0000", 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.719349+0000", 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.719347+0000", 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.719355+0000", 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8469999999999996e-06 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.248 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.719379+0000", 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4164000000000001e-05 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.719388+0000", 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0389999999999993e-06 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.719445+0000", 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6255000000000003e-05 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.249 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.719456+0000", 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1551e-05 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.684262+0000", 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96480605699999999 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.684417+0000", 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015426799999999999 2024-09-21T10:16:52.250 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.684555+0000", 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013847800000000001 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.692892+0000", 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0083367040000000003 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.251 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.692905+0000", 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3354e-05 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693096+0000", 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00019093300000000001 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693145+0000", 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8752999999999999e-05 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693175+0000", 2024-09-21T10:16:52.252 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9879e-05 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226336 6.e 6:7e909d09:::benchmark_data_smithi032_171826_object96713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.738046+0000", 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.46968920399999, 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.95522345600000003, 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.253 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.253 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:52.254 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.254 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226336 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.738046+0000", 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.738046+0000", 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.738044+0000", 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.738051+0000", 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3669999999999999e-06 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.738070+0000", 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8125999999999999e-05 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.738078+0000", 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3329999999999997e-06 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.738129+0000", 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1502999999999998e-05 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.257 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.738138+0000", 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6400000000000003e-06 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.688275+0000", 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.95013691499999997 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.688421+0000", 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014594400000000001 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.258 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.688564+0000", 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014276299999999999 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.692958+0000", 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0043941780000000003 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.692964+0000", 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8590000000000001e-06 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693204+0000", 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024034499999999999 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693240+0000", 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6112999999999998e-05 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693269+0000", 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9261999999999999e-05 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.260 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226341 6.e 6:78f53d21:::benchmark_data_smithi032_171826_object96718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.745338+0000", 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.462397235, 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96607094199999999, 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226341 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.261 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.745338+0000", 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.745338+0000", 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.745336+0000", 2024-09-21T10:16:52.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.745344+0000", 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0409999999999998e-06 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.745364+0000", 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9922999999999999e-05 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.745387+0000", 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3404000000000002e-05 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.745561+0000", 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000173399 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.745587+0000", 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5851999999999999e-05 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.264 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693303+0000", 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.94771645299999996 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693431+0000", 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012799600000000001 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.693551+0000", 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012001 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.702427+0000", 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0088763190000000006 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.702473+0000", 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5451000000000002e-05 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711315+0000", 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0088425280000000005 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711362+0000", 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6603000000000001e-05 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711409+0000", 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6822e-05 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226345 6.e 6:77c0d310:::benchmark_data_smithi032_171826_object96722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.267 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.751020+0000", 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.45671533999999, 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96061449600000004, 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226345 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.751020+0000", 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.751020+0000", 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.751018+0000", 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.751030+0000", 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2126e-05 2024-09-21T10:16:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.751042+0000", 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1532000000000001e-05 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.751050+0000", 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6410000000000002e-06 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.270 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.751102+0000", 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2781000000000002e-05 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.751109+0000", 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3520000000000003e-06 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.697903+0000", 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.946794091 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.697999+0000", 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6014999999999995e-05 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.698138+0000", 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013881399999999999 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.702465+0000", 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0043274180000000004 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.272 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711510+0000", 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0090450870000000003 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711557+0000", 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6823000000000002e-05 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711598+0000", 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1396999999999997e-05 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711634+0000", 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5969000000000001e-05 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226372 6.e 6:74d644d0:::benchmark_data_smithi032_171826_object96749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.781926+0000", 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.42580926599999, 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.92984312999999996, 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.274 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226372 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.781926+0000", 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.781926+0000", 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.781924+0000", 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.781933+0000", 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3789999999999995e-06 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.781952+0000", 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8970999999999998e-05 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.781963+0000", 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0616e-05 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.782025+0000", 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1585000000000001e-05 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.782034+0000", 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5380000000000008e-06 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.702509+0000", 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.92047480599999998 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.702652+0000", 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014280799999999999 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.702739+0000", 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6947000000000006e-05 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711463+0000", 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087244920000000004 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711516+0000", 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2858000000000001e-05 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711690+0000", 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017379200000000001 2024-09-21T10:16:52.279 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711732+0000", 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2654999999999997e-05 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711769+0000", 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6619999999999998e-05 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.280 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226383 6.e 6:7ab8896b:::benchmark_data_smithi032_171826_object96760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.281 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.795525+0000", 2024-09-21T10:16:52.281 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.412210141, 2024-09-21T10:16:52.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.92954523099999997, 2024-09-21T10:16:52.281 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226383 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.795525+0000", 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.795525+0000", 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.795524+0000", 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.795530+0000", 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9039999999999997e-06 2024-09-21T10:16:52.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.795542+0000", 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2396e-05 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.795548+0000", 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9290000000000003e-06 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.795602+0000", 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4182999999999999e-05 2024-09-21T10:16:52.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.795613+0000", 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0716e-05 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.706899+0000", 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.91128640000000005 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.706999+0000", 2024-09-21T10:16:52.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.9675999999999999e-05 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.707103+0000", 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000104455 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711503+0000", 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0043992730000000004 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711523+0000", 2024-09-21T10:16:52.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9941000000000001e-05 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725002+0000", 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013479563 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725034+0000", 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2082999999999998e-05 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725070+0000", 2024-09-21T10:16:52.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6093999999999997e-05 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226386 6.e 6:79bea49e:::benchmark_data_smithi032_171826_object96763:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.800473+0000", 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.40726169999999, 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.92474803900000002, 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.288 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226386 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.800473+0000", 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.800473+0000", 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.800471+0000", 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.800480+0000", 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6219999999999998e-06 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.800500+0000", 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9949000000000002e-05 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.800509+0000", 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.251e-06 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.800570+0000", 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0353999999999998e-05 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.800582+0000", 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2512e-05 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711812+0000", 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.91122953500000003 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.711918+0000", 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000106537 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.712021+0000", 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000102611 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.292 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.724957+0000", 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012935772999999999 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.724968+0000", 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1790000000000001e-05 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725112+0000", 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014368700000000001 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725185+0000", 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3021000000000004e-05 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725222+0000", 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6436000000000001e-05 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226408 6.e 6:71618753:::benchmark_data_smithi032_171826_object96785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.828970+0000", 2024-09-21T10:16:52.294 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.37876533799999, 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.91476596399999999, 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226408 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.828970+0000", 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.295 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.828970+0000", 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.828968+0000", 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.828975+0000", 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3049999999999998e-06 2024-09-21T10:16:52.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.828987+0000", 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2024999999999999e-05 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.828992+0000", 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8649999999999998e-06 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.829534+0000", 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00054162800000000005 2024-09-21T10:16:52.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.829544+0000", 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.7140000000000005e-06 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.720128+0000", 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.89058369699999995 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.720255+0000", 2024-09-21T10:16:52.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012734399999999999 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.720388+0000", 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013264400000000001 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725267+0000", 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0048797950000000001 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.299 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725273+0000", 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7960000000000001e-06 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743651+0000", 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018377747999999999 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743696+0000", 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5262000000000001e-05 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.300 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743736+0000", 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9669000000000003e-05 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226415 6.e 6:7d752d29:::benchmark_data_smithi032_171826_object96792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.838500+0000", 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.36923526300001, 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.91483931600000001, 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.301 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226415 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.838500+0000", 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.838500+0000", 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.302 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.838498+0000", 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.838506+0000", 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1319999999999994e-06 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.838522+0000", 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6240999999999999e-05 2024-09-21T10:16:52.303 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.838530+0000", 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4900000000000003e-06 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.838574+0000", 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4323000000000002e-05 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.838581+0000", 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2230000000000001e-06 2024-09-21T10:16:52.304 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725298+0000", 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.88671688000000004 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725409+0000", 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011086099999999999 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.725517+0000", 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010766499999999999 2024-09-21T10:16:52.305 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743771+0000", 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018254132999999999 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743827+0000", 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6131999999999997e-05 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753255+0000", 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094277549999999995 2024-09-21T10:16:52.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753301+0000", 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5813999999999999e-05 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753339+0000", 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8504000000000003e-05 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226436 6.e 6:739b6211:::benchmark_data_smithi032_171826_object96813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.883024+0000", 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.32471069100001, 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.87044018700000003, 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.309 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226436 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.883024+0000", 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.883024+0000", 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.883023+0000", 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.883028+0000", 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1900000000000003e-06 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.883040+0000", 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1511999999999999e-05 2024-09-21T10:16:52.311 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.883045+0000", 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.186e-06 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.883085+0000", 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0049000000000002e-05 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.883092+0000", 2024-09-21T10:16:52.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8639999999999998e-06 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.729993+0000", 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.84690085599999998 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.730110+0000", 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011702899999999999 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.730235+0000", 2024-09-21T10:16:52.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012536700000000001 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743800+0000", 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013564339 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743854+0000", 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4060000000000001e-05 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753393+0000", 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0095390119999999995 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753431+0000", 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7898000000000002e-05 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753465+0000", 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4093e-05 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.315 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226445 6.e 6:717494be:::benchmark_data_smithi032_171826_object96822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.902327+0000", 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.30540791300001, 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.85120912999999998, 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226445 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.316 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.902327+0000", 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.902327+0000", 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.902326+0000", 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:16:52.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.902332+0000", 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8030000000000002e-06 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.902349+0000", 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6990000000000002e-05 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.902354+0000", 2024-09-21T10:16:52.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5589999999999996e-06 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.902393+0000", 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8994e-05 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.902401+0000", 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8560000000000007e-06 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.734504+0000", 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.83210269400000003 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.734627+0000", 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000123232 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.734764+0000", 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013698200000000001 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743881+0000", 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0091168849999999999 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743909+0000", 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8537000000000001e-05 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753471+0000", 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0095615079999999998 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753505+0000", 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4239000000000002e-05 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753536+0000", 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1231e-05 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44852.0:226466 6.e 6:7dca47a7:::benchmark_data_smithi032_171826_object96843:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:13:28.990277+0000", 2024-09-21T10:16:52.322 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 203.217458389, 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.77358627300000005, 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44852", 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 226466 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.990277+0000", 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.990277+0000", 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.990275+0000", 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.990282+0000", 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6470000000000001e-06 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.990294+0000", 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2330999999999999e-05 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.990302+0000", 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8399999999999995e-06 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.990367+0000", 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4673999999999998e-05 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:28.990388+0000", 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1685e-05 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.743934+0000", 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.75354523399999995 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.744030+0000", 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6175000000000007e-05 2024-09-21T10:16:52.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.744137+0000", 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000107246 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753385+0000", 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0092483010000000004 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.753542+0000", 2024-09-21T10:16:52.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015675699999999999 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.763774+0000", 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010232335 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.763823+0000", 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8565e-05 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.328 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:13:29.763863+0000", 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9983e-05 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:100940 7.1 7:800a03d3:::benchmark_data_smithi032_200324_object100939:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:15:50.316613+0000", 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 61.891122527, 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.083390572999999996, 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:16:52.329 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 100940 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.316613+0000", 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.316613+0000", 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.316611+0000", 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.316668+0000", 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6610999999999997e-05 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.316685+0000", 2024-09-21T10:16:52.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7277000000000001e-05 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.316692+0000", 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5329999999999997e-06 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.316748+0000", 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5296e-05 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.316882+0000", 2024-09-21T10:16:52.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013482500000000001 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.317004+0000", 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000121953 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.320561+0000", 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0035570100000000002 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.323041+0000", 2024-09-21T10:16:52.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024791119999999999 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.399898+0000", 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.076857401000000006 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.399964+0000", 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6248999999999995e-05 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:15:50.400003+0000", 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9038000000000001e-05 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:124603 7.1 7:8c0a7b51:::benchmark_data_smithi032_200324_object124602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:16:13.634667+0000", 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 38.573068401999997, 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15482311400000001, 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.335 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 124603 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.634667+0000", 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.634667+0000", 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.634665+0000", 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.634706+0000", 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0629e-05 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.634720+0000", 2024-09-21T10:16:52.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4715999999999999e-05 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.634728+0000", 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3590000000000002e-06 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.634784+0000", 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6501e-05 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.634913+0000", 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012898799999999999 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.635049+0000", 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000135931 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.642262+0000", 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00721311 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.642290+0000", 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.73e-05 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.789391+0000", 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14710129299999999 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.789450+0000", 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9586999999999997e-05 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.789490+0000", 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9378999999999997e-05 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:124605 7.1 7:8d0b4b15:::benchmark_data_smithi032_200324_object124604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:16:13.638444+0000", 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 38.569290997000003, 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.157282264, 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:52.341 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 124605 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.638444+0000", 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.638444+0000", 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:16:52.342 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.638442+0000", 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.638482+0000", 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9873999999999998e-05 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.638500+0000", 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7439e-05 2024-09-21T10:16:52.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.638508+0000", 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5869999999999993e-06 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.638970+0000", 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00046156300000000001 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.639108+0000", 2024-09-21T10:16:52.344 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013760199999999999 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.639217+0000", 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000108979 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.642412+0000", 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031957029999999998 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.643208+0000", 2024-09-21T10:16:52.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00079599800000000002 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.795567+0000", 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.152358358 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.795683+0000", 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000116111 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:16:52.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:16:52.347 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:13.795726+0000", 2024-09-21T10:16:52.347 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3744e-05 2024-09-21T10:16:52.347 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.347 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.347 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.347 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:16:52.347 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:16:52.347 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:16:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.348 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:52.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:52.449 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:52.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:52.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:52.573 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:52.602 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:52.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:52.704 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:52.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:52.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:52.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:52.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:52.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:52.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.028 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:53.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:53.129 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:53.150 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:53.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:53.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:16:53.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:53.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:53.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:53.472 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:53.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:53.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.611 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:53.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:53.650 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:16:53.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:53.734 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:53.836 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:53.872 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:53.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:53.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:53.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:53.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:54.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.136 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:54.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:54.139 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:53 smithi045 ceph-mon[95976]: pgmap v805: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 15641/283380 objects misplaced (5.519%); 12 MiB/s, 196 objects/s recovering 2024-09-21T10:16:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:53 smithi032 ceph-mon[101299]: pgmap v805: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 15641/283380 objects misplaced (5.519%); 12 MiB/s, 196 objects/s recovering 2024-09-21T10:16:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:53 smithi032 ceph-mon[105748]: pgmap v805: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 15641/283380 objects misplaced (5.519%); 12 MiB/s, 196 objects/s recovering 2024-09-21T10:16:54.237 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:54.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.278 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:54.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:54.379 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:54.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:54.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:54.512 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:54.545 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:54.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:54.646 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:54.675 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:54.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:54.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.857 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:54.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:54.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:54.963 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:54.979 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:16:54.980 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:16:55.022 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:16:55.022 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:55.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:55.056 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:16:55.056 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:55.096 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:16:55.097 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:16:55.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:55.141 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:16:55.141 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:16:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:16:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:54 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:55.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:54 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/575226557' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:55.143 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:16:55.144 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:16:55.144 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:54 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:55.144 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:54 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/575226557' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:55.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:55.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:55.176 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:16:55.176 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:55.210 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:16:55.210 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:55.250 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:16:55.250 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:55.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:16:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:16:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:54 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:16:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:54 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/575226557' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:55.300 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:55.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:55.401 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:55.441 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:55.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:55.543 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:55.576 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:55.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:55.678 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:55.716 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:55.818 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:55.862 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:55.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:55.964 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:55.992 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:55.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:56.094 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:56.101 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:55 smithi032 ceph-mon[105748]: pgmap v806: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 15641/279045 objects misplaced (5.605%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T10:16:56.101 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:55 smithi032 ceph-mon[101299]: pgmap v806: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 15641/279045 objects misplaced (5.605%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T10:16:56.117 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:56.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:56.219 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:56.261 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:55 smithi045 ceph-mon[95976]: pgmap v806: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 15641/279045 objects misplaced (5.605%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T10:16:56.363 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:56.394 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:56.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:56.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:56.525 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:56.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:56.626 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:56.658 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:56.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:56.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:56.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.804 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:56.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:56.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:56.938 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:56.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:57.040 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:57.073 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:57.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:57.175 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:57.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:57.321 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:57.369 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:57.471 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:57.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.505 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:57.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:57.606 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.655 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:57.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:57.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:57.785 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:57.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:57.887 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:57.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:57.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:58.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:58.065 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:58.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:58.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:16:58.165 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:57 smithi032 ceph-mon[105748]: pgmap v807: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 14210/271848 objects misplaced (5.227%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:16:58.166 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:57 smithi032 ceph-mon[101299]: pgmap v807: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 14210/271848 objects misplaced (5.227%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:16:58.167 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:58.193 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:58.215 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:57 smithi045 ceph-mon[95976]: pgmap v807: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 14210/271848 objects misplaced (5.227%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:16:58.295 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:58.321 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:58.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:58.423 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:58.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.457 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:58.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:58.483 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:58.530 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:58.530 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:58.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:58.599 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:58.700 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:58.731 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:58.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:58.833 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.877 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:58.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:58.961 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:16:58.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:59.005 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:59.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:59.106 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:59.146 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:59.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:59.227 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:58 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/892404613' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:59.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:59.259 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:16:59.288 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:59.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:59.312 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:58 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/892404613' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:59.312 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:58 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/892404613' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:59.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:59.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:59.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:59.541 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:59.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:59.616 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:16:59.618 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:16:59.618 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:16:59.619 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-21T10:16:59.619 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:16:59.619 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:16:59.619 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-21T10:16:59.622 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:16:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:16:59] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.51.0" 2024-09-21T10:16:59.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.753 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:16:59.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:59.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:59.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.907 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:16:59.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:00.008 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:59 smithi045 ceph-mon[95976]: pgmap v808: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 14210/268716 objects misplaced (5.288%); 9.5 MiB/s, 152 objects/s recovering 2024-09-21T10:17:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:59 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/892404613' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:16:59 smithi045 ceph-mon[95976]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T10:17:00.063 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:00.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:00.164 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:00.201 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:00.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:00.303 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:00.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:00.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:00.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:00.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:59 smithi032 ceph-mon[101299]: pgmap v808: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 14210/268716 objects misplaced (5.288%); 9.5 MiB/s, 152 objects/s recovering 2024-09-21T10:17:00.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:59 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:00.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/892404613' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:00.459 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:16:59 smithi032 ceph-mon[101299]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T10:17:00.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:59 smithi032 ceph-mon[105748]: pgmap v808: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 14210/268716 objects misplaced (5.288%); 9.5 MiB/s, 152 objects/s recovering 2024-09-21T10:17:00.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:59 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:00.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/892404613' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:00.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:16:59 smithi032 ceph-mon[105748]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T10:17:00.488 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:00.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:00.589 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:00.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.621 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:00.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:00.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:00.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.770 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:00.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:00.871 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:00.913 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:00.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:01.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:01.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.061 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:01.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:01.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:01.191 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:01.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:01.293 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:01.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:01.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:01.425 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:01.457 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:01.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:01.559 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:01.594 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:01.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:01.696 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:01.737 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:01.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:01.839 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:01.867 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:01.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:01.969 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:01.994 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:01.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:02.017 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:02.035 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:17:02.035 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:02.074 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:17:02.074 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:02.096 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:02.120 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:17:02.120 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:02.124 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:02.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:02.160 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:17:02.161 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:02.195 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:17:02.195 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:02.195 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:01 smithi032 ceph-mon[105748]: pgmap v810: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 12805/259629 objects misplaced (4.932%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:17:02.196 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:01 smithi032 ceph-mon[101299]: pgmap v810: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 12805/259629 objects misplaced (4.932%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:17:02.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:02.229 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:17:02.229 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:02.264 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:17:02.264 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:02.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:02.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:01 smithi045 ceph-mon[95976]: pgmap v810: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 12805/259629 objects misplaced (4.932%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:17:02.299 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:17:02.300 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:02.373 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:02.418 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:02.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:02.479 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:17:02.479 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:17:02.479 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:17:02.479 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:17:02.479 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:17:02.520 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:02.573 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:02.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:02.675 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:02.708 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:02.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:02.810 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:02.838 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:02.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:02.940 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:02.972 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:02.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:03.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:03.100 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:03.160 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:17:03.202 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:03.240 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:02 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3246389914' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:03.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.250 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:03.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:03.295 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:02 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3246389914' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:03.296 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:02 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3246389914' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:03.308 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:17:03.351 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.397 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:03.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:03.475 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:03.498 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:03.536 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:03.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:03.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:03.688 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:03.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:03.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:03.831 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:03.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:03.960 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:03.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:04.062 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:04.085 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:04.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:04.187 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:04.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.231 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:04.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:04.271 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:03 smithi032 ceph-mon[101299]: pgmap v811: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 12805/259629 objects misplaced (4.932%); 10 MiB/s, 159 objects/s recovering 2024-09-21T10:17:04.271 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:04.272 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:03 smithi032 ceph-mon[105748]: pgmap v811: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 12805/259629 objects misplaced (4.932%); 10 MiB/s, 159 objects/s recovering 2024-09-21T10:17:04.272 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:03 smithi045 ceph-mon[95976]: pgmap v811: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 12805/259629 objects misplaced (4.932%); 10 MiB/s, 159 objects/s recovering 2024-09-21T10:17:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:04.333 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:04.387 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:04.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:04.489 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:04.524 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:04.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:04.626 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:04.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:04.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:04.763 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:04.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:04.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:04.897 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.924 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:04.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:05.025 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:05.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.062 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:05.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:05.163 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:05.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:05.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:17:05.316 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:05.374 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:05.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:05.477 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:05.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:05.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:05.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.653 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:05.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:05.754 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:05.793 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:05.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:05.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:05.935 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:05.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:05.973 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:17:05.973 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:17:05.973 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233862 5.16 5:6a8e88e5:::benchmark_data_smithi032_140417_object100471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.058459+0000", 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.908274017, 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0285780770000001, 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:05.974 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233862 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058459+0000", 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058459+0000", 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.975 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058457+0000", 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058464+0000", 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.109e-06 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058480+0000", 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5707000000000001e-05 2024-09-21T10:17:05.976 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058488+0000", 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7819999999999994e-06 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058917+0000", 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00042962099999999999 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.977 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.058929+0000", 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1644999999999999e-05 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965364+0000", 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.90643523400000003 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.978 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965388+0000", 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3691999999999999e-05 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.080586+0000", 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.115198087 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.080700+0000", 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011441400000000001 2024-09-21T10:17:05.979 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.081109+0000", 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000408293 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083462+0000", 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0023531110000000002 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.980 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083470+0000", 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9209999999999992e-06 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086650+0000", 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031807570000000002 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086871+0000", 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000221015 2024-09-21T10:17:05.981 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.982 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.982 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.982 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086935+0000", 2024-09-21T10:17:05.982 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3935000000000004e-05 2024-09-21T10:17:05.982 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.982 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086969+0000", 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3460000000000002e-05 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087037+0000", 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7974999999999999e-05 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:05.983 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:05.984 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.984 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.984 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233884 5.16 5:6f45c8b4:::benchmark_data_smithi032_140417_object100493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:05.984 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.074540+0000", 2024-09-21T10:17:05.984 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.89219275699998, 2024-09-21T10:17:05.984 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.015916855, 2024-09-21T10:17:05.984 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:05.984 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233884 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074540+0000", 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.985 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074540+0000", 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074538+0000", 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074545+0000", 2024-09-21T10:17:05.986 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2139999999999999e-06 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074558+0000", 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2448e-05 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.074564+0000", 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6590000000000001e-06 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080211+0000", 2024-09-21T10:17:05.987 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0056471289999999999 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080223+0000", 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1331e-05 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965420+0000", 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.88519726399999998 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965430+0000", 2024-09-21T10:17:05.988 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0386e-05 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083497+0000", 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1180668620000001 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083583+0000", 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6243999999999996e-05 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.083841+0000", 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00025751600000000001 2024-09-21T10:17:05.989 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.086822+0000", 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002981048 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087068+0000", 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024563800000000001 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.990 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087100+0000", 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2191999999999999e-05 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090223+0000", 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00312277 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090262+0000", 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8892e-05 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:05.991 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090311+0000", 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9879999999999997e-05 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090457+0000", 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000145418 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233887 5.16 5:6ae9e2ce:::benchmark_data_smithi032_140417_object100496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.076252+0000", 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.89048045499999, 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0146629210000002, 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233887 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076252+0000", 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076252+0000", 2024-09-21T10:17:05.993 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:05.994 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.994 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.994 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:05.994 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076251+0000", 2024-09-21T10:17:05.994 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:05.994 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.994 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.994 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:05.994 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076257+0000", 2024-09-21T10:17:05.995 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4929999999999998e-06 2024-09-21T10:17:05.995 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.995 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.995 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:05.995 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076271+0000", 2024-09-21T10:17:05.995 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.451e-05 2024-09-21T10:17:05.995 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.995 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.995 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.996 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.076279+0000", 2024-09-21T10:17:05.996 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2259999999999999e-06 2024-09-21T10:17:05.996 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.996 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.996 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:05.996 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080245+0000", 2024-09-21T10:17:05.996 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039659989999999996 2024-09-21T10:17:05.996 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.996 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.997 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.997 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.080252+0000", 2024-09-21T10:17:05.997 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8800000000000008e-06 2024-09-21T10:17:05.997 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.997 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.997 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:05.997 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965457+0000", 2024-09-21T10:17:05.997 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.88520482700000003 2024-09-21T10:17:05.997 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.998 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.998 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.998 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965467+0000", 2024-09-21T10:17:05.998 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.859e-06 2024-09-21T10:17:05.998 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.998 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.998 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:05.998 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.085168+0000", 2024-09-21T10:17:05.998 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.11970049 2024-09-21T10:17:05.999 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.999 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.999 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:05.999 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.085270+0000", 2024-09-21T10:17:05.999 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000102038 2024-09-21T10:17:05.999 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:05.999 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:05.999 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:05.999 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.085489+0000", 2024-09-21T10:17:06.000 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000219848 2024-09-21T10:17:06.000 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.000 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.000 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.000 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087046+0000", 2024-09-21T10:17:06.000 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001556102 2024-09-21T10:17:06.000 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.000 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.001 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.001 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087128+0000", 2024-09-21T10:17:06.001 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2212999999999994e-05 2024-09-21T10:17:06.001 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.001 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.001 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.001 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087159+0000", 2024-09-21T10:17:06.001 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1071000000000001e-05 2024-09-21T10:17:06.001 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.002 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.002 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.002 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090597+0000", 2024-09-21T10:17:06.002 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0034384329999999999 2024-09-21T10:17:06.002 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.002 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.002 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.002 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090768+0000", 2024-09-21T10:17:06.003 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017031500000000001 2024-09-21T10:17:06.003 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.003 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.003 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.003 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090801+0000", 2024-09-21T10:17:06.003 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3090999999999999e-05 2024-09-21T10:17:06.003 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.003 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090915+0000", 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000114504 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:233916 5.16 5:68361781:::benchmark_data_smithi032_140417_object100525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.005 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.104913+0000", 2024-09-21T10:17:06.005 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.86181968099999, 2024-09-21T10:17:06.005 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.986222589, 2024-09-21T10:17:06.005 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.005 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.005 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.005 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.005 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.005 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 233916 2024-09-21T10:17:06.006 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.006 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.006 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.006 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.006 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104913+0000", 2024-09-21T10:17:06.006 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.006 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.006 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.006 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.007 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104913+0000", 2024-09-21T10:17:06.007 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.007 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.007 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.007 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.007 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104911+0000", 2024-09-21T10:17:06.007 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999976 2024-09-21T10:17:06.007 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.007 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.009 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.009 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104918+0000", 2024-09-21T10:17:06.009 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8340000000000001e-06 2024-09-21T10:17:06.009 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.009 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.009 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.009 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104932+0000", 2024-09-21T10:17:06.009 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4585e-05 2024-09-21T10:17:06.009 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.104939+0000", 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9709999999999998e-06 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.105003+0000", 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3701000000000006e-05 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.105024+0000", 2024-09-21T10:17:06.010 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0996e-05 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965500+0000", 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.86047598400000003 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965510+0000", 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8509999999999994e-06 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087182+0000", 2024-09-21T10:17:06.011 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.121672279 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087258+0000", 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5883000000000005e-05 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.087464+0000", 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000206104 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090228+0000", 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002764168 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090519+0000", 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000290513 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090568+0000", 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9011000000000002e-05 2024-09-21T10:17:06.013 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091023+0000", 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00045542999999999999 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091049+0000", 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5609000000000002e-05 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091075+0000", 2024-09-21T10:17:06.014 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.6710999999999999e-05 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091136+0000", 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0202999999999999e-05 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234032 5.16 5:68dad721:::benchmark_data_smithi032_140417_object100641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.70780840800001, 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8323526370000001, 2024-09-21T10:17:06.015 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234032 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.016 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258924+0000", 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258928+0000", 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8249999999999998e-06 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258933+0000", 2024-09-21T10:17:06.017 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258937+0000", 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7699999999999999e-06 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258981+0000", 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3417000000000001e-05 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.018 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.258988+0000", 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6680000000000001e-06 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965537+0000", 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.70654863199999995 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965548+0000", 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0576e-05 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.088645+0000", 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1230973209999999 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.088772+0000", 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012733500000000001 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.088991+0000", 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00021865900000000001 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.020 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090530+0000", 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015394339999999999 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090954+0000", 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00042421599999999999 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.090993+0000", 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8735000000000001e-05 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.021 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091163+0000", 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017015199999999999 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091193+0000", 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9189000000000001e-05 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091216+0000", 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3572000000000001e-05 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091277+0000", 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0851999999999999e-05 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234034 5.16 5:6d7dbd1b:::benchmark_data_smithi032_140417_object100643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.702109474, 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.834288012, 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.023 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234034 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264623+0000", 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264627+0000", 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4209999999999997e-06 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264638+0000", 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1372000000000001e-05 2024-09-21T10:17:06.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264645+0000", 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6409999999999996e-06 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264695+0000", 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9638999999999997e-05 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.264705+0000", 2024-09-21T10:17:06.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0577999999999999e-05 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965574+0000", 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.70086844199999998 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965583+0000", 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8039999999999996e-06 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091314+0000", 2024-09-21T10:17:06.027 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.125730994 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091445+0000", 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000131056 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.091737+0000", 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00029182999999999998 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098585+0000", 2024-09-21T10:17:06.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0068481169999999999 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098634+0000", 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8315000000000001e-05 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098644+0000", 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0589e-05 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.029 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098675+0000", 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0474999999999999e-05 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098715+0000", 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0546000000000001e-05 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098771+0000", 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5798999999999999e-05 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098911+0000", 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000140128 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234086 5.16 5:68c88c0e:::benchmark_data_smithi032_140417_object100695:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.313647+0000", 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.65308557999998, 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.787847314, 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.031 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234086 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313647+0000", 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313647+0000", 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.032 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313646+0000", 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313652+0000", 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9190000000000004e-06 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313663+0000", 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1888999999999999e-05 2024-09-21T10:17:06.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313669+0000", 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3619999999999997e-06 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313709+0000", 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0435999999999997e-05 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.313719+0000", 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3100000000000006e-06 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965613+0000", 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.65189446500000003 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965623+0000", 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6530000000000006e-06 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.093795+0000", 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1281720960000001 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.093926+0000", 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013081799999999999 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.094285+0000", 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00035964900000000003 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098651+0000", 2024-09-21T10:17:06.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0043658539999999997 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099003+0000", 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00035174099999999998 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099064+0000", 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1512000000000003e-05 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099119+0000", 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4803999999999999e-05 2024-09-21T10:17:06.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101348+0000", 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022291329999999999 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101409+0000", 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0467000000000002e-05 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101494+0000", 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5681999999999997e-05 2024-09-21T10:17:06.038 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234124 5.16 5:6c5cf963:::benchmark_data_smithi032_140417_object100733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.351979+0000", 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.61475395100001, 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.754470526, 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.039 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234124 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351979+0000", 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351979+0000", 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351978+0000", 2024-09-21T10:17:06.040 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351982+0000", 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0729999999999998e-06 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351991+0000", 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4260000000000003e-06 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.351996+0000", 2024-09-21T10:17:06.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7219999999999996e-06 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.352067+0000", 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0650999999999998e-05 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.352078+0000", 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0991e-05 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965648+0000", 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.61357046599999998 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965657+0000", 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7949999999999996e-06 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.096078+0000", 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1304205760000001 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.096215+0000", 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000137248 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.096550+0000", 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00033544500000000002 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.098924+0000", 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0023735240000000001 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101244+0000", 2024-09-21T10:17:06.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00231985 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101293+0000", 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8739000000000002e-05 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101542+0000", 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024916499999999998 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106280+0000", 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0047382609999999997 2024-09-21T10:17:06.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106326+0000", 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5986999999999999e-05 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106449+0000", 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000123297 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.046 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234166 5.16 5:6ca4e8a5:::benchmark_data_smithi032_140417_object100775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.387858+0000", 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.57887436700003, 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.718943055, 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234166 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.047 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.048 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:06.048 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:17:06.049 INFO:teuthology.orchestra.run.smithi032.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:17:06.049 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:17:06.049 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387858+0000", 2024-09-21T10:17:06.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387858+0000", 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387857+0000", 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387863+0000", 2024-09-21T10:17:06.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4300000000000003e-06 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387877+0000", 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3878e-05 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387884+0000", 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8839999999999997e-06 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387933+0000", 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8795999999999998e-05 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.387941+0000", 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3939999999999996e-06 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965685+0000", 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57774333 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965693+0000", 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9090000000000006e-06 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099158+0000", 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1334648899999999 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099306+0000", 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000148076 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.053 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.099683+0000", 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00037678700000000001 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106241+0000", 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0065577659999999996 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106512+0000", 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000271301 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.054 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106522+0000", 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0155e-05 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106559+0000", 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6272000000000001e-05 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106608+0000", 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.914e-05 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106668+0000", 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9942000000000003e-05 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106801+0000", 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000133635 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234167 5.16 5:6ce4fd33:::benchmark_data_smithi032_140417_object100776:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.390340+0000", 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.57639290499998, 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.721460698, 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234167 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390340+0000", 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390340+0000", 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390339+0000", 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:06.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390347+0000", 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7729999999999999e-06 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390358+0000", 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1365999999999999e-05 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390363+0000", 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4949999999999999e-06 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390416+0000", 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2979000000000002e-05 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.390428+0000", 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1277000000000001e-05 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.965719+0000", 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57529121699999997 2024-09-21T10:17:06.060 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968363+0000", 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0026442810000000001 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101578+0000", 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.133214739 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101694+0000", 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115877 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.061 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101921+0000", 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000227037 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106824+0000", 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049031949999999999 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106864+0000", 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9711e-05 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.062 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106921+0000", 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7426999999999998e-05 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106960+0000", 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8778e-05 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111484+0000", 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0045242879999999996 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.063 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111533+0000", 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9286999999999998e-05 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111801+0000", 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00026706100000000002 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234170 5.16 5:6c81e8e9:::benchmark_data_smithi032_140417_object100779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.396582+0000", 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.57015071699999, 2024-09-21T10:17:06.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.720816052, 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234170 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396582+0000", 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396582+0000", 2024-09-21T10:17:06.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.066 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.066 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.066 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396581+0000", 2024-09-21T10:17:06.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:06.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.066 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.067 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396585+0000", 2024-09-21T10:17:06.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1250000000000003e-06 2024-09-21T10:17:06.067 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396594+0000", 2024-09-21T10:17:06.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8049999999999996e-06 2024-09-21T10:17:06.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396598+0000", 2024-09-21T10:17:06.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3730000000000003e-06 2024-09-21T10:17:06.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.070 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396637+0000", 2024-09-21T10:17:06.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8633000000000001e-05 2024-09-21T10:17:06.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.396654+0000", 2024-09-21T10:17:06.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6682000000000001e-05 2024-09-21T10:17:06.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968419+0000", 2024-09-21T10:17:06.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57176563899999999 2024-09-21T10:17:06.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.073 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.073 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968433+0000", 2024-09-21T10:17:06.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3628000000000001e-05 2024-09-21T10:17:06.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.103802+0000", 2024-09-21T10:17:06.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.135369106 2024-09-21T10:17:06.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.103915+0000", 2024-09-21T10:17:06.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011266500000000001 2024-09-21T10:17:06.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.075 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.104243+0000", 2024-09-21T10:17:06.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00032865900000000003 2024-09-21T10:17:06.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.076 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.106878+0000", 2024-09-21T10:17:06.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00263496 2024-09-21T10:17:06.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107010+0000", 2024-09-21T10:17:06.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013164200000000001 2024-09-21T10:17:06.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111444+0000", 2024-09-21T10:17:06.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0044338060000000002 2024-09-21T10:17:06.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.079 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.079 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117054+0000", 2024-09-21T10:17:06.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0056105570000000004 2024-09-21T10:17:06.079 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.079 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.079 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.079 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117236+0000", 2024-09-21T10:17:06.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000182058 2024-09-21T10:17:06.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117282+0000", 2024-09-21T10:17:06.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6072000000000002e-05 2024-09-21T10:17:06.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117398+0000", 2024-09-21T10:17:06.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115617 2024-09-21T10:17:06.081 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.081 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.082 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.082 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234173 5.16 5:6d06a077:::benchmark_data_smithi032_140417_object100782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.082 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.399962+0000", 2024-09-21T10:17:06.083 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.566770545, 2024-09-21T10:17:06.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.717757881, 2024-09-21T10:17:06.083 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.083 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.083 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.083 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.083 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.084 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234173 2024-09-21T10:17:06.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.084 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399962+0000", 2024-09-21T10:17:06.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399962+0000", 2024-09-21T10:17:06.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399961+0000", 2024-09-21T10:17:06.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:06.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399966+0000", 2024-09-21T10:17:06.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2000000000000002e-06 2024-09-21T10:17:06.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399977+0000", 2024-09-21T10:17:06.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0924000000000001e-05 2024-09-21T10:17:06.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.399982+0000", 2024-09-21T10:17:06.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1399999999999999e-06 2024-09-21T10:17:06.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.400024+0000", 2024-09-21T10:17:06.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2184999999999997e-05 2024-09-21T10:17:06.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.400032+0000", 2024-09-21T10:17:06.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1709999999999996e-06 2024-09-21T10:17:06.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968461+0000", 2024-09-21T10:17:06.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.56842957699999996 2024-09-21T10:17:06.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968471+0000", 2024-09-21T10:17:06.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8339999999999994e-06 2024-09-21T10:17:06.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107043+0000", 2024-09-21T10:17:06.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1385719139999999 2024-09-21T10:17:06.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107179+0000", 2024-09-21T10:17:06.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013627299999999999 2024-09-21T10:17:06.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.107562+0000", 2024-09-21T10:17:06.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00038266100000000001 2024-09-21T10:17:06.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111861+0000", 2024-09-21T10:17:06.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0042995639999999996 2024-09-21T10:17:06.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111874+0000", 2024-09-21T10:17:06.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2559e-05 2024-09-21T10:17:06.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111932+0000", 2024-09-21T10:17:06.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7599999999999997e-05 2024-09-21T10:17:06.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117167+0000", 2024-09-21T10:17:06.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0052358830000000002 2024-09-21T10:17:06.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117589+0000", 2024-09-21T10:17:06.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000421063 2024-09-21T10:17:06.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117638+0000", 2024-09-21T10:17:06.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9394999999999997e-05 2024-09-21T10:17:06.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117720+0000", 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2128999999999999e-05 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234191 5.16 5:6e4c6783:::benchmark_data_smithi032_140417_object100800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.424857+0000", 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.54187545600001, 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.703213192, 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.099 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234191 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424857+0000", 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424857+0000", 2024-09-21T10:17:06.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424856+0000", 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424862+0000", 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5010000000000001e-06 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424895+0000", 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2545000000000001e-05 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424908+0000", 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3281999999999999e-05 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424983+0000", 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4461000000000004e-05 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.424996+0000", 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3675999999999999e-05 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968498+0000", 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54350163500000004 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968508+0000", 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0185e-05 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.109236+0000", 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.140728183 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.109352+0000", 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115758 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.109580+0000", 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00022780000000000001 2024-09-21T10:17:06.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.114551+0000", 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049710209999999999 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117101+0000", 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0025498420000000001 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117464+0000", 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036382999999999999 2024-09-21T10:17:06.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117550+0000", 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5587999999999994e-05 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.127902+0000", 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010351463999999999 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.127950+0000", 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8401e-05 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128070+0000", 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120559 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234298 5.16 5:6cdfd838:::benchmark_data_smithi032_140417_object100907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.589148+0000", 2024-09-21T10:17:06.107 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.37758459600002, 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5391869969999998, 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234298 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589148+0000", 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589148+0000", 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589146+0000", 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589153+0000", 2024-09-21T10:17:06.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8499999999999996e-06 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589168+0000", 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.447e-05 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589174+0000", 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.003e-06 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589236+0000", 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2528999999999997e-05 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.589249+0000", 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2418999999999999e-05 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968535+0000", 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.37928635399999999 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968545+0000", 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6390000000000004e-06 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.111967+0000", 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.143422419 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.112085+0000", 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000117673 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.112431+0000", 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00034633500000000002 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117112+0000", 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0046805689999999999 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117503+0000", 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00039122000000000001 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117760+0000", 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00025673999999999998 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128141+0000", 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010381095999999999 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128180+0000", 2024-09-21T10:17:06.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9677e-05 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128223+0000", 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2802000000000001e-05 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128335+0000", 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011193599999999999 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234305 5.16 5:6f4d2f3d:::benchmark_data_smithi032_140417_object100914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:03.745057+0000", 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 482.22167539499998, 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3835431599999999, 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234305 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.116 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745057+0000", 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745057+0000", 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745056+0000", 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745061+0000", 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6789999999999996e-06 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745071+0000", 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.031e-05 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745076+0000", 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8060000000000002e-06 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745120+0000", 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3581999999999997e-05 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.745129+0000", 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3030000000000005e-06 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968592+0000", 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.223463151 2024-09-21T10:17:06.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:03.968602+0000", 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0209e-05 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.114594+0000", 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1459913130000001 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.114739+0000", 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014554999999999999 2024-09-21T10:17:06.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.115102+0000", 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036283700000000002 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117514+0000", 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024117330000000001 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117800+0000", 2024-09-21T10:17:06.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00028629399999999998 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.117833+0000", 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2741999999999999e-05 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128400+0000", 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010566862999999999 2024-09-21T10:17:06.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128438+0000", 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8417000000000002e-05 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128488+0000", 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9842999999999997e-05 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128601+0000", 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000112724 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234369 5.e 5:7f9b0c82:::benchmark_data_smithi032_140417_object100978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:05.101285+0000", 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 480.86544768700003, 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0605576990000001, 2024-09-21T10:17:06.124 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234369 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101285+0000", 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101285+0000", 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101284+0000", 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101289+0000", 2024-09-21T10:17:06.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.011e-06 2024-09-21T10:17:06.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101298+0000", 2024-09-21T10:17:06.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9730000000000003e-06 2024-09-21T10:17:06.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.101303+0000", 2024-09-21T10:17:06.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1780000000000002e-06 2024-09-21T10:17:06.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128798+0000", 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.027495070999999999 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128808+0000", 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5869999999999999e-06 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.149057+0000", 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0202487 2024-09-21T10:17:06.129 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.151882+0000", 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002825634 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2,4", 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.152026+0000", 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014369799999999999 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161477+0000", 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094507810000000001 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161535+0000", 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8399000000000001e-05 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161592+0000", 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6943e-05 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161654+0000", 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2346000000000002e-05 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161711+0000", 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6950999999999997e-05 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161843+0000", 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013147699999999999 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234389 5.e 5:7b3804d8:::benchmark_data_smithi032_140417_object100998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:05.120077+0000", 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 480.84665607800002, 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0524572189999999, 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234389 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.133 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120077+0000", 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120077+0000", 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120076+0000", 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120080+0000", 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2039999999999999e-06 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120088+0000", 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6009999999999995e-06 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.120094+0000", 2024-09-21T10:17:06.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6540000000000004e-06 2024-09-21T10:17:06.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128830+0000", 2024-09-21T10:17:06.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087358999999999996 2024-09-21T10:17:06.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.128838+0000", 2024-09-21T10:17:06.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3289999999999994e-06 2024-09-21T10:17:06.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.155438+0000", 2024-09-21T10:17:06.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0265995969999999 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.155529+0000", 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0626999999999998e-05 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2,4", 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.155640+0000", 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000111137 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161604+0000", 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0059642979999999998 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161886+0000", 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00028169099999999998 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161954+0000", 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8204e-05 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.172342+0000", 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010387828999999999 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.172409+0000", 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6789e-05 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.172534+0000", 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000125339 2024-09-21T10:17:06.140 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:234402 5.e 5:7aff253c:::benchmark_data_smithi032_140417_object101011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:05.131926+0000", 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 480.834806754, 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0452540100000001, 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.141 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 234402 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131926+0000", 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131926+0000", 2024-09-21T10:17:06.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131924+0000", 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131931+0000", 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0539999999999997e-06 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131948+0000", 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.683e-05 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.131957+0000", 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4989999999999994e-06 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.132233+0000", 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00027579200000000002 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:05.132244+0000", 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1243999999999999e-05 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.158689+0000", 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0264447269999999 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.158781+0000", 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2371e-05 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,2,4", 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.158857+0000", 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5599000000000006e-05 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.161892+0000", 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0030351319999999998 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.162017+0000", 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000125418 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.168766+0000", 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0067484709999999998 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.176992+0000", 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0082261039999999997 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:06.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.177039+0000", 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7718e-05 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:06.177180+0000", 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014060400000000001 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:238218 5.16 5:6e0183dc:::benchmark_data_smithi032_140417_object104827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:09:10.001585+0000", 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 475.96514810399998, 2024-09-21T10:17:06.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.05259364, 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 238218 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001585+0000", 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.149 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001585+0000", 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001584+0000", 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001588+0000", 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2710000000000001e-06 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.150 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001595+0000", 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5279999999999998e-06 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001601+0000", 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7459999999999997e-06 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.151 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001629+0000", 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7766000000000001e-05 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.001702+0000", 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3644000000000002e-05 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,2,3,7", 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.002088+0000", 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000385728 2024-09-21T10:17:06.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.004328+0000", 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022396650000000001 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.004698+0000", 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00036994000000000003 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.004870+0000", 2024-09-21T10:17:06.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017258400000000001 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.016872+0000", 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012001639 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:10.016887+0000", 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.508e-05 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:11.054100+0000", 2024-09-21T10:17:06.154 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.037213476 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:09:11.054178+0000", 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7967999999999996e-05 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:2406 7.0 7.5fc9b6d0 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:14:44.523357+0000", 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 141.44337564400001, 2024-09-21T10:17:06.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0019831640000001, 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 2406 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.523357+0000", 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.523357+0000", 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.523355+0000", 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.523513+0000", 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015739700000000001 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.157 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.523529+0000", 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6263e-05 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.523538+0000", 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0280000000000006e-06 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.529178+0000", 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0056401940000000003 2024-09-21T10:17:06.158 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.529186+0000", 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4410000000000004e-06 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:45.525331+0000", 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99614562500000003 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:06.159 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:45.525340+0000", 2024-09-21T10:17:06.160 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9050000000000003e-06 2024-09-21T10:17:06.160 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.160 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.160 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.160 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:06.160 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:06.160 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:17:06.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.188 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:06.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:05 smithi032 ceph-mon[101299]: pgmap v812: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 12805/255330 objects misplaced (5.015%); 10 MiB/s, 159 objects/s recovering 2024-09-21T10:17:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:05 smithi032 ceph-mon[105748]: pgmap v812: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 12805/255330 objects misplaced (5.015%); 10 MiB/s, 159 objects/s recovering 2024-09-21T10:17:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:05 smithi045 ceph-mon[95976]: pgmap v812: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 12805/255330 objects misplaced (5.015%); 10 MiB/s, 159 objects/s recovering 2024-09-21T10:17:06.289 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:06.301 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:06.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:06.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:06.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:06.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:06.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:06.577 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:06.612 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:06.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:06.714 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:06.747 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:06.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:06.849 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:06.892 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:06.994 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:07.042 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:07.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:07.053 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:17:07.144 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:07.180 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:07.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:07.282 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:07.314 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:07.395 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:07.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:07.449 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:07.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:07.588 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:07.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:07.669 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:07.689 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:07.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.751 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:07.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:07.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:07.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:07.996 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:08.035 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:08.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:08.136 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:08.174 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:08.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:08.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:07 smithi045 ceph-mon[95976]: pgmap v813: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 11468/248250 objects misplaced (4.620%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:17:08.313 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:08.324 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:07 smithi032 ceph-mon[101299]: pgmap v813: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 11468/248250 objects misplaced (4.620%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:17:08.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:07 smithi032 ceph-mon[105748]: pgmap v813: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 11468/248250 objects misplaced (4.620%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:17:08.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:08.459 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:08.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:08.561 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:08.595 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:08.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:08.697 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:08.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:08.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:08.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:08.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:08.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:09.018 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:09.120 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:09.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:09.257 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:09.296 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:09.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:09.411 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:17:09.411 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:17:09.411 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.411 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:170775 7.c 7:30950232:::benchmark_data_smithi032_200324_object46109:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2024-09-21T10:17:09.411 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:09.394459+0000", 2024-09-21T10:17:09.411 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.013016163000000001, 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013053284, 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 170775 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.394459+0000", 2024-09-21T10:17:09.412 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.413 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.413 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.413 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:09.413 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.394459+0000", 2024-09-21T10:17:09.413 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.413 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.413 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.414 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:09.414 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.394457+0000", 2024-09-21T10:17:09.414 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:17:09.414 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.414 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.414 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:09.415 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.394463+0000", 2024-09-21T10:17:09.415 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.404e-06 2024-09-21T10:17:09.415 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.415 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.415 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:09.415 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.394474+0000", 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0436e-05 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.394481+0000", 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9500000000000004e-06 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.404607+0000", 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010126636 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.416 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.404734+0000", 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012640399999999999 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.404857+0000", 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012335100000000001 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.417 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:170770 7.4 e276/248 7:2a2b0cab:::benchmark_data_smithi032_200324_object46104:head v 276'10636, mlcod=276'10631)", 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:09.395435+0000", 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.012039886, 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012167631, 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.395435+0000", 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.418 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.395435+0000", 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.395434+0000", 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.395441+0000", 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4659999999999996e-06 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.419 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.395452+0000", 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0789e-05 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.395458+0000", 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3840000000000002e-06 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.420 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.405672+0000", 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.01021364 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.405691+0000", 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9213000000000001e-05 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:170777 7.b e276/256 7:d690a718:::benchmark_data_smithi032_200324_object46111:head v 276'10672, mlcod=276'10668)", 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:09.399708+0000", 2024-09-21T10:17:09.421 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0077670769999999998, 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0079564309999999999, 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.399708+0000", 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.399708+0000", 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.422 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.399708+0000", 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.399712+0000", 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1479999999999999e-06 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.399719+0000", 2024-09-21T10:17:09.423 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1300000000000003e-06 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.399723+0000", 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6070000000000001e-06 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.403628+0000", 2024-09-21T10:17:09.424 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039043569999999998 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.403650+0000", 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1767999999999999e-05 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.425 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:170760 7.4 e276/248 7:2ac52134:::benchmark_data_smithi032_200324_object46094:head v 276'10633, mlcod=276'10631)", 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:09.384706+0000", 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.022769142999999999, 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.023016726000000001, 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.384706+0000", 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.426 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.427 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.427 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:09.427 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.384706+0000", 2024-09-21T10:17:09.427 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.427 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.427 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.428 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:09.428 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.384705+0000", 2024-09-21T10:17:09.428 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:09.428 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.428 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.428 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.384712+0000", 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5139999999999998e-06 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.384725+0000", 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3687e-05 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:09.429 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.384732+0000", 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1160000000000001e-06 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.384801+0000", 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9122999999999995e-05 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.384822+0000", 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0628e-05 2024-09-21T10:17:09.430 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:170766 7.4 e276/248 7:29e70d56:::benchmark_data_smithi032_200324_object46100:head v 276'10634, mlcod=276'10631)", 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:09.385100+0000", 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.022375248, 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.022682425999999999, 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.385100+0000", 2024-09-21T10:17:09.431 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.385100+0000", 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.385099+0000", 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.432 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.385105+0000", 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6760000000000004e-06 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.385114+0000", 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4939999999999992e-06 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.385120+0000", 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0460000000000002e-06 2024-09-21T10:17:09.433 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.389397+0000", 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0042762629999999998 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.389416+0000", 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9066999999999999e-05 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.434 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:170769 7.4 e276/248 7:295c9065:::benchmark_data_smithi032_200324_object46103:head v 276'10635, mlcod=276'10631)", 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:09.390730+0000", 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.016745322, 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.017115600000000002, 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.390730+0000", 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.435 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.390730+0000", 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.390729+0000", 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.390736+0000", 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8959999999999997e-06 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.436 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.390747+0000", 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1714e-05 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.390753+0000", 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2269999999999998e-06 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.396139+0000", 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0053859850000000003 2024-09-21T10:17:09.437 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:09.396159+0000", 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9783e-05 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 6 2024-09-21T10:17:09.438 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:17:09.440 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:09.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:09.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:09.587 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:09.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:09.689 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:09.724 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:09.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:09.826 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:09.858 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:09.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:09.883 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:17:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:17:09] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.51.0" 2024-09-21T10:17:09.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:10.010 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:10.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:10.112 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:10.130 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:10 smithi045 ceph-mon[95976]: pgmap v814: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 11468/245160 objects misplaced (4.678%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:17:10.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.174 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:10.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:17:10.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:10.176 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:10 smithi032 ceph-mon[101299]: pgmap v814: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 11468/245160 objects misplaced (4.678%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:17:10.176 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:10 smithi032 ceph-mon[105748]: pgmap v814: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 11468/245160 objects misplaced (4.678%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:17:10.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:10.313 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:10.415 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:10.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.452 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:10.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:10.513 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:10.553 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:10.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.599 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:10.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:10.700 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:10.722 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:17:10.723 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:10.731 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:10.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:10.768 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:17:10.768 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:10.813 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:17:10.813 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:10.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:10.861 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:17:10.861 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:10.871 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:10.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:10.896 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:17:10.897 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:10.936 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:17:10.937 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:10.970 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:17:10.970 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:10.973 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.003 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:11.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:11.004 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:17:11.004 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:11.105 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:11.138 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:11.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:11 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1300048520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:11.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:11 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1300048520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:11.240 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:11.277 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:11.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:11 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1300048520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:11.379 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:11.419 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:11.521 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:11.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:11.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:11.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:11.704 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:11.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:11.805 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:11.838 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:11.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:11.895 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:17:11.895 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:17:11.895 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:17:11.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:17:11.940 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:11.963 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:11.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:12 smithi045 ceph-mon[95976]: pgmap v815: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 10042/236100 objects misplaced (4.253%); 14 MiB/s, 219 objects/s recovering 2024-09-21T10:17:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:12 smithi045 ceph-mon[95976]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T10:17:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:12 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2488827095' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:12.064 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:12.087 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:12.133 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:12.188 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:12 smithi032 ceph-mon[105748]: pgmap v815: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 10042/236100 objects misplaced (4.253%); 14 MiB/s, 219 objects/s recovering 2024-09-21T10:17:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:12 smithi032 ceph-mon[105748]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T10:17:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:12 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2488827095' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:12 smithi032 ceph-mon[101299]: pgmap v815: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 10042/236100 objects misplaced (4.253%); 14 MiB/s, 219 objects/s recovering 2024-09-21T10:17:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:12 smithi032 ceph-mon[101299]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T10:17:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:12 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2488827095' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:12.212 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:12.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:12.314 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:12.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:12.386 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:12.386 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:12.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:12.464 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:12.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:12.566 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:12.589 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:12.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:12.612 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:17:12.691 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:12.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.730 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:12.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:12.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:12.883 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:12.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:12.984 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:13.026 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:13.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:13.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.176 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:13.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:13.183 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:13 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:13.183 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2488827095' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:13.183 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:13 smithi045 ceph-mon[95976]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T10:17:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:13 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2488827095' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:13 smithi032 ceph-mon[105748]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T10:17:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:13 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2488827095' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:13 smithi032 ceph-mon[101299]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T10:17:13.223 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:17:13.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:13.315 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:13.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:13.450 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:13.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:13.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:13.554 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:13.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.613 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:13.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:13.713 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:13.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:13.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:13.879 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:13.912 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:13.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:14.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:14.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.045 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:14.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:14.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:17:14.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:14.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.197 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:14.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:14.198 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:14 smithi032 ceph-mon[101299]: pgmap v817: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 10042/236100 objects misplaced (4.253%); 11 MiB/s, 170 objects/s recovering 2024-09-21T10:17:14.199 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:14 smithi032 ceph-mon[105748]: pgmap v817: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 10042/236100 objects misplaced (4.253%); 11 MiB/s, 170 objects/s recovering 2024-09-21T10:17:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:14 smithi045 ceph-mon[95976]: pgmap v817: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 10042/236100 objects misplaced (4.253%); 11 MiB/s, 170 objects/s recovering 2024-09-21T10:17:14.298 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:14.349 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:14.451 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:14.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:14.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:14.605 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:14.650 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:14.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:14.752 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:14.789 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:14.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:14.928 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.947 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:14.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:15.048 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:15.084 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:15.188 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:17:15.189 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:17:15.190 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:15.238 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:15.340 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:15.374 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:15.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:15.476 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.514 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:15.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:15.523 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:15.523 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:15.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:15.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:15.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:15.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:15.805 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:15.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:15.907 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:15.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.943 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:15.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:16.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:16.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.086 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:16.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:16.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:16.200 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:17:16.220 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:16 smithi045 ceph-mon[95976]: pgmap v819: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 10042/232224 objects misplaced (4.324%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T10:17:16.224 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:16.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:17:16.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:16.369 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:16.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:16.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:16 smithi032 ceph-mon[101299]: pgmap v819: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 10042/232224 objects misplaced (4.324%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T10:17:16.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:16 smithi032 ceph-mon[105748]: pgmap v819: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 10042/232224 objects misplaced (4.324%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T10:17:16.472 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:16.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.524 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:16.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:16.625 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:16.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:16.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:16.695 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:16.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:16.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.819 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:16.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:16.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:16.962 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:17.064 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:17.119 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:17.221 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:17.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:17.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:17.268 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:17 smithi032 ceph-mon[105748]: from='client.55096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:17:17.268 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:17 smithi032 ceph-mon[101299]: from='client.55096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:17:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:17 smithi045 ceph-mon[95976]: from='client.55096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:17:17.313 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":819,"stamp":"2024-09-21T10:17:14.397934+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5075900413,"num_objects":77408,"num_object_clones":0,"num_object_copies":232224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10042,"num_objects_unfound":0,"num_objects_dirty":77408,"num_whiteouts":0,"num_read":6391,"num_read_kb":13846,"num_write":173071,"num_write_kb":7993474,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6342,"num_bytes_recovered":422318656,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130530,"ondisk_log_size":130530,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18751408,"kb_used_data":15257128,"kb_used_omap":35,"kb_used_meta":3494108,"kb_avail":731209808,"statfs":{"total":767960285184,"available":748758843392,"internally_reserved":0,"allocated":15623299072,"data_stored":15612969865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":3577967310},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,5,3,8,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-350093312,"num_objects":-5342,"num_object_clones":0,"num_object_copies":-16026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1426,"num_objects_unfound":0,"num_objects_dirty":-5342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5342,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":911,"num_bytes_recovered":59703296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001975"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":430,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.613293+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:17:12.613293+0000","last_peered":"2024-09-21T10:17:12.613293+0000","last_clean":"2024-09-21T10:17:12.613293+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:17:12.613293+0000","last_undegraded":"2024-09-21T10:17:12.613293+0000","last_fullsized":"2024-09-21T10:17:12.613293+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":246,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.932120+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:16:50.932120+0000","last_peered":"2024-09-21T10:16:50.932120+0000","last_clean":"2024-09-21T10:16:50.932120+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:16:50.932120+0000","last_undegraded":"2024-09-21T10:16:50.932120+0000","last_fullsized":"2024-09-21T10:16:50.932120+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":35,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.964788+0000","last_change":"2024-09-21T10:14:45.529072+0000","last_active":"2024-09-21T10:16:58.964788+0000","last_peered":"2024-09-21T10:16:58.964788+0000","last_clean":"2024-09-21T10:16:58.964788+0000","last_became_active":"2024-09-21T10:14:45.528270+0000","last_became_peered":"2024-09-21T10:14:45.528270+0000","last_unstale":"2024-09-21T10:16:58.964788+0000","last_undegraded":"2024-09-21T10:16:58.964788+0000","last_fullsized":"2024-09-21T10:16:58.964788+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:00.326719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":178,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.931323+0000","last_change":"2024-09-21T10:13:09.583319+0000","last_active":"2024-09-21T10:16:50.931323+0000","last_peered":"2024-09-21T10:16:50.931323+0000","last_clean":"2024-09-21T10:16:50.931323+0000","last_became_active":"2024-09-21T10:13:09.582953+0000","last_became_peered":"2024-09-21T10:13:09.582953+0000","last_unstale":"2024-09-21T10:16:50.931323+0000","last_undegraded":"2024-09-21T10:16:50.931323+0000","last_fullsized":"2024-09-21T10:16:50.931323+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:13.630031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":258,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.613559+0000","last_change":"2024-09-21T10:09:35.958207+0000","last_active":"2024-09-21T10:17:12.613559+0000","last_peered":"2024-09-21T10:17:12.613559+0000","last_clean":"2024-09-21T10:17:12.613559+0000","last_became_active":"2024-09-21T10:09:35.957860+0000","last_became_peered":"2024-09-21T10:09:35.957860+0000","last_unstale":"2024-09-21T10:17:12.613559+0000","last_undegraded":"2024-09-21T10:17:12.613559+0000","last_fullsized":"2024-09-21T10:17:12.613559+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:37.767930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":229,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.931454+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:16:50.931454+0000","last_peered":"2024-09-21T10:16:50.931454+0000","last_clean":"2024-09-21T10:16:50.931454+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:16:50.931454+0000","last_undegraded":"2024-09-21T10:16:50.931454+0000","last_fullsized":"2024-09-21T10:16:50.931454+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":437,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.614018+0000","last_change":"2024-09-21T10:14:20.761360+0000","last_active":"2024-09-21T10:17:12.614018+0000","last_peered":"2024-09-21T10:17:12.614018+0000","last_clean":"2024-09-21T10:17:12.614018+0000","last_became_active":"2024-09-21T10:14:20.760941+0000","last_became_peered":"2024-09-21T10:14:20.760941+0000","last_unstale":"2024-09-21T10:17:12.614018+0000","last_undegraded":"2024-09-21T10:17:12.614018+0000","last_fullsized":"2024-09-21T10:17:12.614018+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:27.842256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":294,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.928957+0000","last_change":"2024-09-21T10:14:45.537300+0000","last_active":"2024-09-21T10:16:50.928957+0000","last_peered":"2024-09-21T10:16:50.928957+0000","last_clean":"2024-09-21T10:16:50.928957+0000","last_became_active":"2024-09-21T10:14:45.527447+0000","last_became_peered":"2024-09-21T10:14:45.527447+0000","last_unstale":"2024-09-21T10:16:50.928957+0000","last_undegraded":"2024-09-21T10:16:50.928957+0000","last_fullsized":"2024-09-21T10:16:50.928957+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:33.652169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"276'10705","reported_seq":11066,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:17:10.012181+0000","last_change":"2024-09-21T10:14:49.554195+0000","last_active":"2024-09-21T10:17:10.012181+0000","last_peered":"2024-09-21T10:17:10.012181+0000","last_clean":"2024-09-21T10:17:10.012181+0000","last_became_active":"2024-09-21T10:14:49.551887+0000","last_became_peered":"2024-09-21T10:14:49.551887+0000","last_unstale":"2024-09-21T10:17:10.012181+0000","last_undegraded":"2024-09-21T10:17:10.012181+0000","last_fullsized":"2024-09-21T10:17:10.012181+0000","mapping_epoch":256,"log_start":"263'4100","ondisk_log_start":"263'4100","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6605,"log_dups_size":0,"ondisk_log_size":6605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:26.931226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325255168,"num_objects":4963,"num_object_clones":0,"num_object_copies":14889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10705,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":9043968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":284,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.931683+0000","last_change":"2024-09-21T10:13:09.586457+0000","last_active":"2024-09-21T10:16:50.931683+0000","last_peered":"2024-09-21T10:16:50.931683+0000","last_clean":"2024-09-21T10:16:50.931683+0000","last_became_active":"2024-09-21T10:13:09.574624+0000","last_became_peered":"2024-09-21T10:13:09.574624+0000","last_unstale":"2024-09-21T10:16:50.931683+0000","last_undegraded":"2024-09-21T10:16:50.931683+0000","last_fullsized":"2024-09-21T10:16:50.931683+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:39:00.778086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":35,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:54.754927+0000","last_change":"2024-09-21T10:14:45.530298+0000","last_active":"2024-09-21T10:16:54.754927+0000","last_peered":"2024-09-21T10:16:54.754927+0000","last_clean":"2024-09-21T10:16:54.754927+0000","last_became_active":"2024-09-21T10:14:45.529912+0000","last_became_peered":"2024-09-21T10:14:45.529912+0000","last_unstale":"2024-09-21T10:16:54.754927+0000","last_undegraded":"2024-09-21T10:16:54.754927+0000","last_fullsized":"2024-09-21T10:16:54.754927+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:50.567684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"278'10837","reported_seq":11235,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:13.048231+0000","last_change":"2024-09-21T10:14:49.553650+0000","last_active":"2024-09-21T10:17:13.048231+0000","last_peered":"2024-09-21T10:17:13.048231+0000","last_clean":"2024-09-21T10:17:13.048231+0000","last_became_active":"2024-09-21T10:14:49.552007+0000","last_became_peered":"2024-09-21T10:14:49.552007+0000","last_unstale":"2024-09-21T10:17:13.048231+0000","last_undegraded":"2024-09-21T10:17:13.048231+0000","last_fullsized":"2024-09-21T10:17:13.048231+0000","mapping_epoch":256,"log_start":"261'3800","ondisk_log_start":"261'3800","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:43:27.336777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318832640,"num_objects":4865,"num_object_clones":0,"num_object_copies":14595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10837,"num_write_kb":502464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":311,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.932017+0000","last_change":"2024-09-21T10:13:09.596445+0000","last_active":"2024-09-21T10:16:50.932017+0000","last_peered":"2024-09-21T10:16:50.932017+0000","last_clean":"2024-09-21T10:16:50.932017+0000","last_became_active":"2024-09-21T10:13:09.582665+0000","last_became_peered":"2024-09-21T10:13:09.582665+0000","last_unstale":"2024-09-21T10:16:50.932017+0000","last_undegraded":"2024-09-21T10:16:50.932017+0000","last_fullsized":"2024-09-21T10:16:50.932017+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:23:47.510351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":277,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.911396+0000","last_change":"2024-09-21T10:13:09.578819+0000","last_active":"2024-09-21T10:16:50.911396+0000","last_peered":"2024-09-21T10:16:50.911396+0000","last_clean":"2024-09-21T10:16:50.911396+0000","last_became_active":"2024-09-21T10:13:09.576707+0000","last_became_peered":"2024-09-21T10:13:09.576707+0000","last_unstale":"2024-09-21T10:16:50.911396+0000","last_undegraded":"2024-09-21T10:16:50.911396+0000","last_fullsized":"2024-09-21T10:16:50.911396+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:19.175846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"276'10717","reported_seq":10798,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:17:09.991472+0000","last_change":"2024-09-21T10:14:32.771740+0000","last_active":"2024-09-21T10:17:09.991472+0000","last_peered":"2024-09-21T10:17:09.991472+0000","last_clean":"2024-09-21T10:17:09.991472+0000","last_became_active":"2024-09-21T10:14:32.771298+0000","last_became_peered":"2024-09-21T10:14:32.771298+0000","last_unstale":"2024-09-21T10:17:09.991472+0000","last_undegraded":"2024-09-21T10:17:09.991472+0000","last_fullsized":"2024-09-21T10:17:09.991472+0000","mapping_epoch":248,"log_start":"262'4100","ondisk_log_start":"262'4100","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:27.287728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324075520,"num_objects":4945,"num_object_clones":0,"num_object_copies":14835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10717,"num_write_kb":501184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":256,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.931880+0000","last_change":"2024-09-21T10:14:45.533171+0000","last_active":"2024-09-21T10:16:50.931880+0000","last_peered":"2024-09-21T10:16:50.931880+0000","last_clean":"2024-09-21T10:16:50.931880+0000","last_became_active":"2024-09-21T10:14:45.527709+0000","last_became_peered":"2024-09-21T10:14:45.527709+0000","last_unstale":"2024-09-21T10:16:50.931880+0000","last_undegraded":"2024-09-21T10:16:50.931880+0000","last_fullsized":"2024-09-21T10:16:50.931880+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:33.989738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":262,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.928972+0000","last_change":"2024-09-21T10:13:09.578358+0000","last_active":"2024-09-21T10:16:50.928972+0000","last_peered":"2024-09-21T10:16:50.928972+0000","last_clean":"2024-09-21T10:16:50.928972+0000","last_became_active":"2024-09-21T10:13:09.577273+0000","last_became_peered":"2024-09-21T10:13:09.577273+0000","last_unstale":"2024-09-21T10:16:50.928972+0000","last_undegraded":"2024-09-21T10:16:50.928972+0000","last_fullsized":"2024-09-21T10:16:50.928972+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:17.971932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"278'10832","reported_seq":10910,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:13.892332+0000","last_change":"2024-09-21T10:14:28.438058+0000","last_active":"2024-09-21T10:17:13.892332+0000","last_peered":"2024-09-21T10:17:13.892332+0000","last_clean":"2024-09-21T10:17:13.892332+0000","last_became_active":"2024-09-21T10:14:28.437772+0000","last_became_peered":"2024-09-21T10:14:28.437772+0000","last_unstale":"2024-09-21T10:17:13.892332+0000","last_undegraded":"2024-09-21T10:17:13.892332+0000","last_fullsized":"2024-09-21T10:17:13.892332+0000","mapping_epoch":245,"log_start":"257'800","ondisk_log_start":"257'800","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":10032,"log_dups_size":0,"ondisk_log_size":10032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:14.437061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308412416,"num_objects":4706,"num_object_clones":0,"num_object_copies":14118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10832,"num_write_kb":497216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":244,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.940219+0000","last_change":"2024-09-21T10:03:53.843578+0000","last_active":"2024-09-21T10:16:50.940219+0000","last_peered":"2024-09-21T10:16:50.940219+0000","last_clean":"2024-09-21T10:16:50.940219+0000","last_became_active":"2024-09-21T10:03:53.843166+0000","last_became_peered":"2024-09-21T10:03:53.843166+0000","last_unstale":"2024-09-21T10:16:50.940219+0000","last_undegraded":"2024-09-21T10:16:50.940219+0000","last_fullsized":"2024-09-21T10:16:50.940219+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:25.336164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":24,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.967036+0000","last_change":"2024-09-21T10:16:35.252681+0000","last_active":"2024-09-21T10:16:58.967036+0000","last_peered":"2024-09-21T10:16:58.967036+0000","last_clean":"2024-09-21T10:16:58.967036+0000","last_became_active":"2024-09-21T10:16:35.252333+0000","last_became_peered":"2024-09-21T10:16:35.252333+0000","last_unstale":"2024-09-21T10:16:58.967036+0000","last_undegraded":"2024-09-21T10:16:58.967036+0000","last_fullsized":"2024-09-21T10:16:58.967036+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:43.211846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"278'10906","reported_seq":10984,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:13.059511+0000","last_change":"2024-09-21T10:14:28.441282+0000","last_active":"2024-09-21T10:17:13.059511+0000","last_peered":"2024-09-21T10:17:13.059511+0000","last_clean":"2024-09-21T10:17:13.059511+0000","last_became_active":"2024-09-21T10:14:28.440767+0000","last_became_peered":"2024-09-21T10:14:28.440767+0000","last_unstale":"2024-09-21T10:17:13.059511+0000","last_undegraded":"2024-09-21T10:17:13.059511+0000","last_fullsized":"2024-09-21T10:17:13.059511+0000","mapping_epoch":245,"log_start":"261'3900","ondisk_log_start":"261'3900","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:56:18.651559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304480256,"num_objects":4646,"num_object_clones":0,"num_object_copies":13938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10906,"num_write_kb":497664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":447,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.613169+0000","last_change":"2024-09-21T10:13:09.594302+0000","last_active":"2024-09-21T10:17:12.613169+0000","last_peered":"2024-09-21T10:17:12.613169+0000","last_clean":"2024-09-21T10:17:12.613169+0000","last_became_active":"2024-09-21T10:13:09.576948+0000","last_became_peered":"2024-09-21T10:13:09.576948+0000","last_unstale":"2024-09-21T10:17:12.613169+0000","last_undegraded":"2024-09-21T10:17:12.613169+0000","last_fullsized":"2024-09-21T10:17:12.613169+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:10.360060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":243,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.912145+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:16:50.912145+0000","last_peered":"2024-09-21T10:16:50.912145+0000","last_clean":"2024-09-21T10:16:50.912145+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:16:50.912145+0000","last_undegraded":"2024-09-21T10:16:50.912145+0000","last_fullsized":"2024-09-21T10:16:50.912145+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"278'10959","reported_seq":11382,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:13.943376+0000","last_change":"2024-09-21T10:14:49.553783+0000","last_active":"2024-09-21T10:17:13.943376+0000","last_peered":"2024-09-21T10:17:13.943376+0000","last_clean":"2024-09-21T10:17:13.943376+0000","last_became_active":"2024-09-21T10:14:49.552803+0000","last_became_peered":"2024-09-21T10:14:49.552803+0000","last_unstale":"2024-09-21T10:17:13.943376+0000","last_undegraded":"2024-09-21T10:17:13.943376+0000","last_fullsized":"2024-09-21T10:17:13.943376+0000","mapping_epoch":256,"log_start":"257'900","ondisk_log_start":"257'900","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":10059,"log_dups_size":0,"ondisk_log_size":10059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:56:20.507993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308871168,"num_objects":4713,"num_object_clones":0,"num_object_copies":14139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10959,"num_write_kb":501504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":166,"num_bytes_recovered":10878976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":263,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.929067+0000","last_change":"2024-09-21T10:13:09.578151+0000","last_active":"2024-09-21T10:16:50.929067+0000","last_peered":"2024-09-21T10:16:50.929067+0000","last_clean":"2024-09-21T10:16:50.929067+0000","last_became_active":"2024-09-21T10:13:09.577233+0000","last_became_peered":"2024-09-21T10:13:09.577233+0000","last_unstale":"2024-09-21T10:16:50.929067+0000","last_undegraded":"2024-09-21T10:16:50.929067+0000","last_fullsized":"2024-09-21T10:16:50.929067+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:00.627590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":39,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.911037+0000","last_change":"2024-09-21T10:14:32.771000+0000","last_active":"2024-09-21T10:16:50.911037+0000","last_peered":"2024-09-21T10:16:50.911037+0000","last_clean":"2024-09-21T10:16:50.911037+0000","last_became_active":"2024-09-21T10:14:32.770562+0000","last_became_peered":"2024-09-21T10:14:32.770562+0000","last_unstale":"2024-09-21T10:16:50.911037+0000","last_undegraded":"2024-09-21T10:16:50.911037+0000","last_fullsized":"2024-09-21T10:16:50.911037+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:47.956518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"276'10648","reported_seq":10712,"reported_epoch":276,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:17:09.691954+0000","last_change":"2024-09-21T10:16:36.457453+0000","last_active":"2024-09-21T10:17:09.691954+0000","last_peered":"2024-09-21T10:17:09.691954+0000","last_clean":"2024-09-21T10:16:34.205791+0000","last_became_active":"2024-09-21T10:16:36.384021+0000","last_became_peered":"2024-09-21T10:16:36.384021+0000","last_unstale":"2024-09-21T10:17:09.691954+0000","last_undegraded":"2024-09-21T10:17:09.691954+0000","last_fullsized":"2024-09-21T10:17:09.691954+0000","mapping_epoch":270,"log_start":"258'1800","ondisk_log_start":"258'1800","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":8848,"log_dups_size":0,"ondisk_log_size":8848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:05:24.740053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321060888,"num_objects":4900,"num_object_clones":0,"num_object_copies":14700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4898,"num_objects_unfound":0,"num_objects_dirty":4900,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10648,"num_write_kb":497473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,3],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":4900}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":245,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.911642+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:16:50.911642+0000","last_peered":"2024-09-21T10:16:50.911642+0000","last_clean":"2024-09-21T10:16:50.911642+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:16:50.911642+0000","last_undegraded":"2024-09-21T10:16:50.911642+0000","last_fullsized":"2024-09-21T10:16:50.911642+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":261,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.911248+0000","last_change":"2024-09-21T10:13:09.579444+0000","last_active":"2024-09-21T10:16:50.911248+0000","last_peered":"2024-09-21T10:16:50.911248+0000","last_clean":"2024-09-21T10:16:50.911248+0000","last_became_active":"2024-09-21T10:13:09.579162+0000","last_became_peered":"2024-09-21T10:13:09.579162+0000","last_unstale":"2024-09-21T10:16:50.911248+0000","last_undegraded":"2024-09-21T10:16:50.911248+0000","last_fullsized":"2024-09-21T10:16:50.911248+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.051244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"276'10545","reported_seq":10586,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:17:09.761656+0000","last_change":"2024-09-21T10:14:28.439503+0000","last_active":"2024-09-21T10:17:09.761656+0000","last_peered":"2024-09-21T10:17:09.761656+0000","last_clean":"2024-09-21T10:17:09.761656+0000","last_became_active":"2024-09-21T10:14:28.438413+0000","last_became_peered":"2024-09-21T10:14:28.438413+0000","last_unstale":"2024-09-21T10:17:09.761656+0000","last_undegraded":"2024-09-21T10:17:09.761656+0000","last_fullsized":"2024-09-21T10:17:09.761656+0000","mapping_epoch":245,"log_start":"258'1700","ondisk_log_start":"258'1700","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":8845,"log_dups_size":0,"ondisk_log_size":8845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:24.386364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320536576,"num_objects":4891,"num_object_clones":0,"num_object_copies":14673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10545,"num_write_kb":493952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":432,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.612998+0000","last_change":"2024-09-21T10:13:09.582151+0000","last_active":"2024-09-21T10:17:12.612998+0000","last_peered":"2024-09-21T10:17:12.612998+0000","last_clean":"2024-09-21T10:17:12.612998+0000","last_became_active":"2024-09-21T10:13:09.580224+0000","last_became_peered":"2024-09-21T10:13:09.580224+0000","last_unstale":"2024-09-21T10:17:12.612998+0000","last_undegraded":"2024-09-21T10:17:12.612998+0000","last_fullsized":"2024-09-21T10:17:12.612998+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:15.490461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":133,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.970390+0000","last_change":"2024-09-21T10:13:09.586643+0000","last_active":"2024-09-21T10:16:58.970390+0000","last_peered":"2024-09-21T10:16:58.970390+0000","last_clean":"2024-09-21T10:16:58.970390+0000","last_became_active":"2024-09-21T10:13:09.586283+0000","last_became_peered":"2024-09-21T10:13:09.586283+0000","last_unstale":"2024-09-21T10:16:58.970390+0000","last_undegraded":"2024-09-21T10:16:58.970390+0000","last_fullsized":"2024-09-21T10:16:58.970390+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:20.002373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"276'10807","reported_seq":11190,"reported_epoch":276,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:17:09.739984+0000","last_change":"2024-09-21T10:16:36.492510+0000","last_active":"2024-09-21T10:17:09.739984+0000","last_peered":"2024-09-21T10:17:09.739984+0000","last_clean":"2024-09-21T10:16:34.220422+0000","last_became_active":"2024-09-21T10:16:36.430443+0000","last_became_peered":"2024-09-21T10:16:36.430443+0000","last_unstale":"2024-09-21T10:17:09.739984+0000","last_undegraded":"2024-09-21T10:17:09.739984+0000","last_fullsized":"2024-09-21T10:17:09.739984+0000","mapping_epoch":270,"log_start":"260'3100","ondisk_log_start":"260'3100","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7707,"log_dups_size":0,"ondisk_log_size":7707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:22.221602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322633728,"num_objects":4923,"num_object_clones":0,"num_object_copies":14769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4923,"num_objects_unfound":0,"num_objects_dirty":4923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10807,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[7,3,5],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":4923}],"blocked_by":[],"up_primary":0,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":646,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.931446+0000","last_change":"2024-09-21T10:13:12.573143+0000","last_active":"2024-09-21T10:16:50.931446+0000","last_peered":"2024-09-21T10:16:50.931446+0000","last_clean":"2024-09-21T10:16:50.931446+0000","last_became_active":"2024-09-21T10:13:12.571751+0000","last_became_peered":"2024-09-21T10:13:12.571751+0000","last_unstale":"2024-09-21T10:16:50.931446+0000","last_undegraded":"2024-09-21T10:16:50.931446+0000","last_fullsized":"2024-09-21T10:16:50.931446+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:28.804119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":595,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.941951+0000","last_change":"2024-09-21T10:03:19.764688+0000","last_active":"2024-09-21T10:16:50.941951+0000","last_peered":"2024-09-21T10:16:50.941951+0000","last_clean":"2024-09-21T10:16:50.941951+0000","last_became_active":"2024-09-21T10:03:19.764221+0000","last_became_peered":"2024-09-21T10:03:19.764221+0000","last_unstale":"2024-09-21T10:16:50.941951+0000","last_undegraded":"2024-09-21T10:16:50.941951+0000","last_fullsized":"2024-09-21T10:16:50.941951+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:43.287804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":394,"num_read_kb":631,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":379,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.613130+0000","last_change":"2024-09-21T10:14:45.527731+0000","last_active":"2024-09-21T10:17:12.613130+0000","last_peered":"2024-09-21T10:17:12.613130+0000","last_clean":"2024-09-21T10:17:12.613130+0000","last_became_active":"2024-09-21T10:14:45.527330+0000","last_became_peered":"2024-09-21T10:14:45.527330+0000","last_unstale":"2024-09-21T10:17:12.613130+0000","last_undegraded":"2024-09-21T10:17:12.613130+0000","last_fullsized":"2024-09-21T10:17:12.613130+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:03:48.411604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":229,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.929249+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:16:50.929249+0000","last_peered":"2024-09-21T10:16:50.929249+0000","last_clean":"2024-09-21T10:16:50.929249+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:16:50.929249+0000","last_undegraded":"2024-09-21T10:16:50.929249+0000","last_fullsized":"2024-09-21T10:16:50.929249+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"276'10654","reported_seq":10695,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:17:09.739508+0000","last_change":"2024-09-21T10:14:28.440396+0000","last_active":"2024-09-21T10:17:09.739508+0000","last_peered":"2024-09-21T10:17:09.739508+0000","last_clean":"2024-09-21T10:17:09.739508+0000","last_became_active":"2024-09-21T10:14:28.440137+0000","last_became_peered":"2024-09-21T10:14:28.440137+0000","last_unstale":"2024-09-21T10:17:09.739508+0000","last_undegraded":"2024-09-21T10:17:09.739508+0000","last_fullsized":"2024-09-21T10:17:09.739508+0000","mapping_epoch":245,"log_start":"260'2900","ondisk_log_start":"260'2900","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7754,"log_dups_size":0,"ondisk_log_size":7754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:33:10.456362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317849600,"num_objects":4850,"num_object_clones":0,"num_object_copies":14550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10654,"num_write_kb":496128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":912,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.931672+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:16:50.931672+0000","last_peered":"2024-09-21T10:16:50.931672+0000","last_clean":"2024-09-21T10:16:50.931672+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:16:50.931672+0000","last_undegraded":"2024-09-21T10:16:50.931672+0000","last_fullsized":"2024-09-21T10:16:50.931672+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":24,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.966859+0000","last_change":"2024-09-21T10:16:35.252630+0000","last_active":"2024-09-21T10:16:58.966859+0000","last_peered":"2024-09-21T10:16:58.966859+0000","last_clean":"2024-09-21T10:16:58.966859+0000","last_became_active":"2024-09-21T10:16:35.252279+0000","last_became_peered":"2024-09-21T10:16:35.252279+0000","last_unstale":"2024-09-21T10:16:58.966859+0000","last_undegraded":"2024-09-21T10:16:58.966859+0000","last_fullsized":"2024-09-21T10:16:58.966859+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:12.918989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":133,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.970295+0000","last_change":"2024-09-21T10:13:09.580568+0000","last_active":"2024-09-21T10:16:58.970295+0000","last_peered":"2024-09-21T10:16:58.970295+0000","last_clean":"2024-09-21T10:16:58.970295+0000","last_became_active":"2024-09-21T10:13:09.579053+0000","last_became_peered":"2024-09-21T10:13:09.579053+0000","last_unstale":"2024-09-21T10:16:58.970295+0000","last_undegraded":"2024-09-21T10:16:58.970295+0000","last_fullsized":"2024-09-21T10:16:58.970295+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:39.206612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"276'10647","reported_seq":10686,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:17:09.694413+0000","last_change":"2024-09-21T10:14:32.642821+0000","last_active":"2024-09-21T10:17:09.694413+0000","last_peered":"2024-09-21T10:17:09.694413+0000","last_clean":"2024-09-21T10:17:09.694413+0000","last_became_active":"2024-09-21T10:14:32.642404+0000","last_became_peered":"2024-09-21T10:14:32.642404+0000","last_unstale":"2024-09-21T10:17:09.694413+0000","last_undegraded":"2024-09-21T10:17:09.694413+0000","last_fullsized":"2024-09-21T10:17:09.694413+0000","mapping_epoch":248,"log_start":"259'2700","ondisk_log_start":"259'2700","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7947,"log_dups_size":0,"ondisk_log_size":7947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:02.051331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315555840,"num_objects":4815,"num_object_clones":0,"num_object_copies":14445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10647,"num_write_kb":494784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":979,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.637907+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:17:12.637907+0000","last_peered":"2024-09-21T10:17:12.637907+0000","last_clean":"2024-09-21T10:17:12.637907+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:17:12.637907+0000","last_undegraded":"2024-09-21T10:17:12.637907+0000","last_fullsized":"2024-09-21T10:17:12.637907+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":134,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:59.004017+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:16:59.004017+0000","last_peered":"2024-09-21T10:16:59.004017+0000","last_clean":"2024-09-21T10:16:59.004017+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:16:59.004017+0000","last_undegraded":"2024-09-21T10:16:59.004017+0000","last_fullsized":"2024-09-21T10:16:59.004017+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":133,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:59.006175+0000","last_change":"2024-09-21T10:13:09.578776+0000","last_active":"2024-09-21T10:16:59.006175+0000","last_peered":"2024-09-21T10:16:59.006175+0000","last_clean":"2024-09-21T10:16:59.006175+0000","last_became_active":"2024-09-21T10:13:09.578462+0000","last_became_peered":"2024-09-21T10:13:09.578462+0000","last_unstale":"2024-09-21T10:16:59.006175+0000","last_undegraded":"2024-09-21T10:16:59.006175+0000","last_fullsized":"2024-09-21T10:16:59.006175+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:19.035767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"276'10657","reported_seq":21461,"reported_epoch":276,"state":"active+remapped+backfilling","last_fresh":"2024-09-21T10:17:09.778757+0000","last_change":"2024-09-21T10:16:36.495847+0000","last_active":"2024-09-21T10:17:09.778757+0000","last_peered":"2024-09-21T10:17:09.778757+0000","last_clean":"2024-09-21T10:16:34.225685+0000","last_became_active":"2024-09-21T10:16:36.384107+0000","last_became_peered":"2024-09-21T10:16:36.384107+0000","last_unstale":"2024-09-21T10:17:09.778757+0000","last_undegraded":"2024-09-21T10:17:09.778757+0000","last_fullsized":"2024-09-21T10:17:09.778757+0000","mapping_epoch":270,"log_start":"258'1800","ondisk_log_start":"258'1800","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":8857,"log_dups_size":0,"ondisk_log_size":8857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:11:18.445278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318439424,"num_objects":4859,"num_object_clones":0,"num_object_copies":14577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":221,"num_objects_unfound":0,"num_objects_dirty":4859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10657,"num_write_kb":496512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5376,"num_bytes_recovered":352321536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,3,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":4859}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1069,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.940107+0000","last_change":"2024-09-21T10:13:12.560467+0000","last_active":"2024-09-21T10:16:50.940107+0000","last_peered":"2024-09-21T10:16:50.940107+0000","last_clean":"2024-09-21T10:16:50.940107+0000","last_became_active":"2024-09-21T10:13:12.559882+0000","last_became_peered":"2024-09-21T10:13:12.559882+0000","last_unstale":"2024-09-21T10:16:50.940107+0000","last_undegraded":"2024-09-21T10:16:50.940107+0000","last_fullsized":"2024-09-21T10:16:50.940107+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:33.267760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":170,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.637998+0000","last_change":"2024-09-21T10:16:35.253445+0000","last_active":"2024-09-21T10:17:12.637998+0000","last_peered":"2024-09-21T10:17:12.637998+0000","last_clean":"2024-09-21T10:17:12.637998+0000","last_became_active":"2024-09-21T10:16:35.253121+0000","last_became_peered":"2024-09-21T10:16:35.253121+0000","last_unstale":"2024-09-21T10:17:12.637998+0000","last_undegraded":"2024-09-21T10:17:12.637998+0000","last_fullsized":"2024-09-21T10:17:12.637998+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:36.816904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":133,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:59.009493+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:16:59.009493+0000","last_peered":"2024-09-21T10:16:59.009493+0000","last_clean":"2024-09-21T10:16:59.009493+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:16:59.009493+0000","last_undegraded":"2024-09-21T10:16:59.009493+0000","last_fullsized":"2024-09-21T10:16:59.009493+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"276'10545","reported_seq":10586,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:17:09.700424+0000","last_change":"2024-09-21T10:14:28.437169+0000","last_active":"2024-09-21T10:17:09.700424+0000","last_peered":"2024-09-21T10:17:09.700424+0000","last_clean":"2024-09-21T10:17:09.700424+0000","last_became_active":"2024-09-21T10:14:28.436873+0000","last_became_peered":"2024-09-21T10:14:28.436873+0000","last_unstale":"2024-09-21T10:17:09.700424+0000","last_undegraded":"2024-09-21T10:17:09.700424+0000","last_fullsized":"2024-09-21T10:17:09.700424+0000","mapping_epoch":245,"log_start":"259'2600","ondisk_log_start":"259'2600","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7945,"log_dups_size":0,"ondisk_log_size":7945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:16.516429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321323008,"num_objects":4903,"num_object_clones":0,"num_object_copies":14709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10545,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":260,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.940307+0000","last_change":"2024-09-21T10:03:53.843567+0000","last_active":"2024-09-21T10:16:50.940307+0000","last_peered":"2024-09-21T10:16:50.940307+0000","last_clean":"2024-09-21T10:16:50.940307+0000","last_became_active":"2024-09-21T10:03:53.843155+0000","last_became_peered":"2024-09-21T10:03:53.843155+0000","last_unstale":"2024-09-21T10:16:50.940307+0000","last_undegraded":"2024-09-21T10:16:50.940307+0000","last_fullsized":"2024-09-21T10:16:50.940307+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:07.232159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":742,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:59.009729+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:16:59.009729+0000","last_peered":"2024-09-21T10:16:59.009729+0000","last_clean":"2024-09-21T10:16:59.009729+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:16:59.009729+0000","last_undegraded":"2024-09-21T10:16:59.009729+0000","last_fullsized":"2024-09-21T10:16:59.009729+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":147,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.970071+0000","last_change":"2024-09-21T10:14:45.531907+0000","last_active":"2024-09-21T10:16:58.970071+0000","last_peered":"2024-09-21T10:16:58.970071+0000","last_clean":"2024-09-21T10:16:58.970071+0000","last_became_active":"2024-09-21T10:14:45.531536+0000","last_became_peered":"2024-09-21T10:14:45.531536+0000","last_unstale":"2024-09-21T10:16:58.970071+0000","last_undegraded":"2024-09-21T10:16:58.970071+0000","last_fullsized":"2024-09-21T10:16:58.970071+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:18:52.155338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"276'10845","reported_seq":11191,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:17:09.740641+0000","last_change":"2024-09-21T10:14:48.563829+0000","last_active":"2024-09-21T10:17:09.740641+0000","last_peered":"2024-09-21T10:17:09.740641+0000","last_clean":"2024-09-21T10:17:09.740641+0000","last_became_active":"2024-09-21T10:14:48.549468+0000","last_became_peered":"2024-09-21T10:14:48.549468+0000","last_unstale":"2024-09-21T10:17:09.740641+0000","last_undegraded":"2024-09-21T10:17:09.740641+0000","last_fullsized":"2024-09-21T10:17:09.740641+0000","mapping_epoch":255,"log_start":"257'800","ondisk_log_start":"257'800","created":245,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":10045,"log_dups_size":0,"ondisk_log_size":10045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:08.229536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319488000,"num_objects":4875,"num_object_clones":0,"num_object_copies":14625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10845,"num_write_kb":503040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":140,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:59.009644+0000","last_change":"2024-09-21T10:14:45.530390+0000","last_active":"2024-09-21T10:16:59.009644+0000","last_peered":"2024-09-21T10:16:59.009644+0000","last_clean":"2024-09-21T10:16:59.009644+0000","last_became_active":"2024-09-21T10:14:45.530054+0000","last_became_peered":"2024-09-21T10:14:45.530054+0000","last_unstale":"2024-09-21T10:16:59.009644+0000","last_undegraded":"2024-09-21T10:16:59.009644+0000","last_fullsized":"2024-09-21T10:16:59.009644+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:42:49.775110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":758,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.931929+0000","last_change":"2024-09-21T10:13:09.604050+0000","last_active":"2024-09-21T10:16:50.931929+0000","last_peered":"2024-09-21T10:16:50.931929+0000","last_clean":"2024-09-21T10:16:50.931929+0000","last_became_active":"2024-09-21T10:13:09.583593+0000","last_became_peered":"2024-09-21T10:13:09.583593+0000","last_unstale":"2024-09-21T10:16:50.931929+0000","last_undegraded":"2024-09-21T10:16:50.931929+0000","last_fullsized":"2024-09-21T10:16:50.931929+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:35.420105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":24,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.966941+0000","last_change":"2024-09-21T10:16:35.250102+0000","last_active":"2024-09-21T10:16:58.966941+0000","last_peered":"2024-09-21T10:16:58.966941+0000","last_clean":"2024-09-21T10:16:58.966941+0000","last_became_active":"2024-09-21T10:16:35.249779+0000","last_became_peered":"2024-09-21T10:16:35.249779+0000","last_unstale":"2024-09-21T10:16:58.966941+0000","last_undegraded":"2024-09-21T10:16:58.966941+0000","last_fullsized":"2024-09-21T10:16:58.966941+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:37.423305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"278'10881","reported_seq":10959,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:13.060961+0000","last_change":"2024-09-21T10:14:28.438913+0000","last_active":"2024-09-21T10:17:13.060961+0000","last_peered":"2024-09-21T10:17:13.060961+0000","last_clean":"2024-09-21T10:17:13.060961+0000","last_became_active":"2024-09-21T10:14:28.438598+0000","last_became_peered":"2024-09-21T10:14:28.438598+0000","last_unstale":"2024-09-21T10:17:13.060961+0000","last_undegraded":"2024-09-21T10:17:13.060961+0000","last_fullsized":"2024-09-21T10:17:13.060961+0000","mapping_epoch":245,"log_start":"261'3900","ondisk_log_start":"261'3900","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6981,"log_dups_size":0,"ondisk_log_size":6981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:29:24.765631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316080128,"num_objects":4823,"num_object_clones":0,"num_object_copies":14469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10881,"num_write_kb":502528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":437,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.614150+0000","last_change":"2024-09-21T10:16:35.254311+0000","last_active":"2024-09-21T10:17:12.614150+0000","last_peered":"2024-09-21T10:17:12.614150+0000","last_clean":"2024-09-21T10:17:12.614150+0000","last_became_active":"2024-09-21T10:16:35.250010+0000","last_became_peered":"2024-09-21T10:16:35.250010+0000","last_unstale":"2024-09-21T10:17:12.614150+0000","last_undegraded":"2024-09-21T10:17:12.614150+0000","last_fullsized":"2024-09-21T10:17:12.614150+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:13.306874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":737,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.970206+0000","last_change":"2024-09-21T10:13:12.569309+0000","last_active":"2024-09-21T10:16:58.970206+0000","last_peered":"2024-09-21T10:16:58.970206+0000","last_clean":"2024-09-21T10:16:58.970206+0000","last_became_active":"2024-09-21T10:13:12.568325+0000","last_became_peered":"2024-09-21T10:13:12.568325+0000","last_unstale":"2024-09-21T10:16:58.970206+0000","last_undegraded":"2024-09-21T10:16:58.970206+0000","last_fullsized":"2024-09-21T10:16:58.970206+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:18.314566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":433,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.612898+0000","last_change":"2024-09-21T10:13:09.579331+0000","last_active":"2024-09-21T10:17:12.612898+0000","last_peered":"2024-09-21T10:17:12.612898+0000","last_clean":"2024-09-21T10:17:12.612898+0000","last_became_active":"2024-09-21T10:13:09.578777+0000","last_became_peered":"2024-09-21T10:13:09.578777+0000","last_unstale":"2024-09-21T10:17:12.612898+0000","last_undegraded":"2024-09-21T10:17:12.612898+0000","last_fullsized":"2024-09-21T10:17:12.612898+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:50:34.425710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"278'10763","reported_seq":10841,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:13.037452+0000","last_change":"2024-09-21T10:14:28.439188+0000","last_active":"2024-09-21T10:17:13.037452+0000","last_peered":"2024-09-21T10:17:13.037452+0000","last_clean":"2024-09-21T10:17:13.037452+0000","last_became_active":"2024-09-21T10:14:28.438763+0000","last_became_peered":"2024-09-21T10:14:28.438763+0000","last_unstale":"2024-09-21T10:17:13.037452+0000","last_undegraded":"2024-09-21T10:17:13.037452+0000","last_fullsized":"2024-09-21T10:17:13.037452+0000","mapping_epoch":245,"log_start":"261'3700","ondisk_log_start":"261'3700","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7063,"log_dups_size":0,"ondisk_log_size":7063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:31.383586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308215808,"num_objects":4703,"num_object_clones":0,"num_object_copies":14109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10763,"num_write_kb":494912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":137,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.970067+0000","last_change":"2024-09-21T10:13:09.592519+0000","last_active":"2024-09-21T10:16:58.970067+0000","last_peered":"2024-09-21T10:16:58.970067+0000","last_clean":"2024-09-21T10:16:58.970067+0000","last_became_active":"2024-09-21T10:13:09.587615+0000","last_became_peered":"2024-09-21T10:13:09.587615+0000","last_unstale":"2024-09-21T10:16:58.970067+0000","last_undegraded":"2024-09-21T10:16:58.970067+0000","last_fullsized":"2024-09-21T10:16:58.970067+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:22:43.216612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":520,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.929712+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:16:50.929712+0000","last_peered":"2024-09-21T10:16:50.929712+0000","last_clean":"2024-09-21T10:16:50.929712+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:16:50.929712+0000","last_undegraded":"2024-09-21T10:16:50.929712+0000","last_fullsized":"2024-09-21T10:16:50.929712+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":229,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.911472+0000","last_change":"2024-09-21T09:50:23.755982+0000","last_active":"2024-09-21T10:16:50.911472+0000","last_peered":"2024-09-21T10:16:50.911472+0000","last_clean":"2024-09-21T10:16:50.911472+0000","last_became_active":"2024-09-21T09:50:23.755429+0000","last_became_peered":"2024-09-21T09:50:23.755429+0000","last_unstale":"2024-09-21T10:16:50.911472+0000","last_undegraded":"2024-09-21T10:16:50.911472+0000","last_fullsized":"2024-09-21T10:16:50.911472+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:25.865716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":270,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.911615+0000","last_change":"2024-09-21T10:15:21.038463+0000","last_active":"2024-09-21T10:16:50.911615+0000","last_peered":"2024-09-21T10:16:50.911615+0000","last_clean":"2024-09-21T10:16:50.911615+0000","last_became_active":"2024-09-21T10:15:21.038061+0000","last_became_peered":"2024-09-21T10:15:21.038061+0000","last_unstale":"2024-09-21T10:16:50.911615+0000","last_undegraded":"2024-09-21T10:16:50.911615+0000","last_fullsized":"2024-09-21T10:16:50.911615+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.241295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":252,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.912276+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:16:50.912276+0000","last_peered":"2024-09-21T10:16:50.912276+0000","last_clean":"2024-09-21T10:16:50.912276+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:16:50.912276+0000","last_undegraded":"2024-09-21T10:16:50.912276+0000","last_fullsized":"2024-09-21T10:16:50.912276+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":24,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.939953+0000","last_change":"2024-09-21T10:15:21.039385+0000","last_active":"2024-09-21T10:16:50.939953+0000","last_peered":"2024-09-21T10:16:50.939953+0000","last_clean":"2024-09-21T10:16:50.939953+0000","last_became_active":"2024-09-21T10:15:21.039073+0000","last_became_peered":"2024-09-21T10:15:21.039073+0000","last_unstale":"2024-09-21T10:16:50.939953+0000","last_undegraded":"2024-09-21T10:16:50.939953+0000","last_fullsized":"2024-09-21T10:16:50.939953+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:11.160836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":247,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:54.749749+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:16:54.749749+0000","last_peered":"2024-09-21T10:16:54.749749+0000","last_clean":"2024-09-21T10:16:54.749749+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:16:54.749749+0000","last_undegraded":"2024-09-21T10:16:54.749749+0000","last_fullsized":"2024-09-21T10:16:54.749749+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":35,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.967124+0000","last_change":"2024-09-21T10:14:45.528895+0000","last_active":"2024-09-21T10:16:58.967124+0000","last_peered":"2024-09-21T10:16:58.967124+0000","last_clean":"2024-09-21T10:16:58.967124+0000","last_became_active":"2024-09-21T10:14:45.528608+0000","last_became_peered":"2024-09-21T10:14:45.528608+0000","last_unstale":"2024-09-21T10:16:58.967124+0000","last_undegraded":"2024-09-21T10:16:58.967124+0000","last_fullsized":"2024-09-21T10:16:58.967124+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:48.887817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":364,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.613131+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:17:12.613131+0000","last_peered":"2024-09-21T10:17:12.613131+0000","last_clean":"2024-09-21T10:17:12.613131+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:17:12.613131+0000","last_undegraded":"2024-09-21T10:17:12.613131+0000","last_fullsized":"2024-09-21T10:17:12.613131+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":24,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.967314+0000","last_change":"2024-09-21T10:16:35.248480+0000","last_active":"2024-09-21T10:16:58.967314+0000","last_peered":"2024-09-21T10:16:58.967314+0000","last_clean":"2024-09-21T10:16:58.967314+0000","last_became_active":"2024-09-21T10:16:35.248125+0000","last_became_peered":"2024-09-21T10:16:35.248125+0000","last_unstale":"2024-09-21T10:16:58.967314+0000","last_undegraded":"2024-09-21T10:16:58.967314+0000","last_fullsized":"2024-09-21T10:16:58.967314+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:12.431615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":443,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.612865+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:17:12.612865+0000","last_peered":"2024-09-21T10:17:12.612865+0000","last_clean":"2024-09-21T10:17:12.612865+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:17:12.612865+0000","last_undegraded":"2024-09-21T10:17:12.612865+0000","last_fullsized":"2024-09-21T10:17:12.612865+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":271,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.929511+0000","last_change":"2024-09-21T10:13:09.573620+0000","last_active":"2024-09-21T10:16:50.929511+0000","last_peered":"2024-09-21T10:16:50.929511+0000","last_clean":"2024-09-21T10:16:50.929511+0000","last_became_active":"2024-09-21T10:13:09.573216+0000","last_became_peered":"2024-09-21T10:13:09.573216+0000","last_unstale":"2024-09-21T10:16:50.929511+0000","last_undegraded":"2024-09-21T10:16:50.929511+0000","last_fullsized":"2024-09-21T10:16:50.929511+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:40.253863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":453,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.612961+0000","last_change":"2024-09-21T10:13:09.581932+0000","last_active":"2024-09-21T10:17:12.612961+0000","last_peered":"2024-09-21T10:17:12.612961+0000","last_clean":"2024-09-21T10:17:12.612961+0000","last_became_active":"2024-09-21T10:13:09.580798+0000","last_became_peered":"2024-09-21T10:13:09.580798+0000","last_unstale":"2024-09-21T10:17:12.612961+0000","last_undegraded":"2024-09-21T10:17:12.612961+0000","last_fullsized":"2024-09-21T10:17:12.612961+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:05.563826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":24,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.969309+0000","last_change":"2024-09-21T10:16:35.250798+0000","last_active":"2024-09-21T10:16:58.969309+0000","last_peered":"2024-09-21T10:16:58.969309+0000","last_clean":"2024-09-21T10:16:58.969309+0000","last_became_active":"2024-09-21T10:16:35.250461+0000","last_became_peered":"2024-09-21T10:16:35.250461+0000","last_unstale":"2024-09-21T10:16:58.969309+0000","last_undegraded":"2024-09-21T10:16:58.969309+0000","last_fullsized":"2024-09-21T10:16:58.969309+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:37.340319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":140,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.970141+0000","last_change":"2024-09-21T10:14:32.454735+0000","last_active":"2024-09-21T10:16:58.970141+0000","last_peered":"2024-09-21T10:16:58.970141+0000","last_clean":"2024-09-21T10:16:58.970141+0000","last_became_active":"2024-09-21T10:14:32.454475+0000","last_became_peered":"2024-09-21T10:14:32.454475+0000","last_unstale":"2024-09-21T10:16:58.970141+0000","last_undegraded":"2024-09-21T10:16:58.970141+0000","last_fullsized":"2024-09-21T10:16:58.970141+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:59.191649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":436,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.613463+0000","last_change":"2024-09-21T10:14:20.761163+0000","last_active":"2024-09-21T10:17:12.613463+0000","last_peered":"2024-09-21T10:17:12.613463+0000","last_clean":"2024-09-21T10:17:12.613463+0000","last_became_active":"2024-09-21T10:14:20.760814+0000","last_became_peered":"2024-09-21T10:14:20.760814+0000","last_unstale":"2024-09-21T10:17:12.613463+0000","last_undegraded":"2024-09-21T10:17:12.613463+0000","last_fullsized":"2024-09-21T10:17:12.613463+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:44.679743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":277,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.911102+0000","last_change":"2024-09-21T10:13:09.577994+0000","last_active":"2024-09-21T10:16:50.911102+0000","last_peered":"2024-09-21T10:16:50.911102+0000","last_clean":"2024-09-21T10:16:50.911102+0000","last_became_active":"2024-09-21T10:13:09.577311+0000","last_became_peered":"2024-09-21T10:13:09.577311+0000","last_unstale":"2024-09-21T10:16:50.911102+0000","last_undegraded":"2024-09-21T10:16:50.911102+0000","last_fullsized":"2024-09-21T10:16:50.911102+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:27.532202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":449,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.638080+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:17:12.638080+0000","last_peered":"2024-09-21T10:17:12.638080+0000","last_clean":"2024-09-21T10:17:12.638080+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:17:12.638080+0000","last_undegraded":"2024-09-21T10:17:12.638080+0000","last_fullsized":"2024-09-21T10:17:12.638080+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":157,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.911506+0000","last_change":"2024-09-21T10:13:09.578314+0000","last_active":"2024-09-21T10:16:50.911506+0000","last_peered":"2024-09-21T10:16:50.911506+0000","last_clean":"2024-09-21T10:16:50.911506+0000","last_became_active":"2024-09-21T10:13:09.577570+0000","last_became_peered":"2024-09-21T10:13:09.577570+0000","last_unstale":"2024-09-21T10:16:50.911506+0000","last_undegraded":"2024-09-21T10:16:50.911506+0000","last_fullsized":"2024-09-21T10:16:50.911506+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:21.299713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.974638+0000","last_change":"2024-09-21T10:13:09.577841+0000","last_active":"2024-09-21T10:16:58.974638+0000","last_peered":"2024-09-21T10:16:58.974638+0000","last_clean":"2024-09-21T10:16:58.974638+0000","last_became_active":"2024-09-21T10:13:09.576793+0000","last_became_peered":"2024-09-21T10:13:09.576793+0000","last_unstale":"2024-09-21T10:16:58.974638+0000","last_undegraded":"2024-09-21T10:16:58.974638+0000","last_fullsized":"2024-09-21T10:16:58.974638+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:12.718520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":271,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.940130+0000","last_change":"2024-09-21T10:13:09.589654+0000","last_active":"2024-09-21T10:16:50.940130+0000","last_peered":"2024-09-21T10:16:50.940130+0000","last_clean":"2024-09-21T10:16:50.940130+0000","last_became_active":"2024-09-21T10:13:09.574438+0000","last_became_peered":"2024-09-21T10:13:09.574438+0000","last_unstale":"2024-09-21T10:16:50.940130+0000","last_undegraded":"2024-09-21T10:16:50.940130+0000","last_fullsized":"2024-09-21T10:16:50.940130+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:12.504161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":48,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.973676+0000","last_change":"2024-09-21T10:16:35.252818+0000","last_active":"2024-09-21T10:16:58.973676+0000","last_peered":"2024-09-21T10:16:58.973676+0000","last_clean":"2024-09-21T10:16:58.973676+0000","last_became_active":"2024-09-21T10:16:35.252497+0000","last_became_peered":"2024-09-21T10:16:35.252497+0000","last_unstale":"2024-09-21T10:16:58.973676+0000","last_undegraded":"2024-09-21T10:16:58.973676+0000","last_fullsized":"2024-09-21T10:16:58.973676+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:30.018576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":230,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.940240+0000","last_change":"2024-09-21T09:50:22.396992+0000","last_active":"2024-09-21T10:16:50.940240+0000","last_peered":"2024-09-21T10:16:50.940240+0000","last_clean":"2024-09-21T10:16:50.940240+0000","last_became_active":"2024-09-21T09:50:22.396720+0000","last_became_peered":"2024-09-21T09:50:22.396720+0000","last_unstale":"2024-09-21T10:16:50.940240+0000","last_undegraded":"2024-09-21T10:16:50.940240+0000","last_fullsized":"2024-09-21T10:16:50.940240+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:05.459513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":463,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.613491+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:17:12.613491+0000","last_peered":"2024-09-21T10:17:12.613491+0000","last_clean":"2024-09-21T10:17:12.613491+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:17:12.613491+0000","last_undegraded":"2024-09-21T10:17:12.613491+0000","last_fullsized":"2024-09-21T10:17:12.613491+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":424,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-09-21T10:17:12.613628+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:17:12.613628+0000","last_peered":"2024-09-21T10:17:12.613628+0000","last_clean":"2024-09-21T10:17:12.613628+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:17:12.613628+0000","last_undegraded":"2024-09-21T10:17:12.613628+0000","last_fullsized":"2024-09-21T10:17:12.613628+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":261,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.973996+0000","last_change":"2024-09-21T10:13:09.577388+0000","last_active":"2024-09-21T10:16:58.973996+0000","last_peered":"2024-09-21T10:16:58.973996+0000","last_clean":"2024-09-21T10:16:58.973996+0000","last_became_active":"2024-09-21T10:13:09.577027+0000","last_became_peered":"2024-09-21T10:13:09.577027+0000","last_unstale":"2024-09-21T10:16:58.973996+0000","last_undegraded":"2024-09-21T10:16:58.973996+0000","last_fullsized":"2024-09-21T10:16:58.973996+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:49.522609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":41,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-09-21T10:16:50.940308+0000","last_change":"2024-09-21T10:14:45.536046+0000","last_active":"2024-09-21T10:16:50.940308+0000","last_peered":"2024-09-21T10:16:50.940308+0000","last_clean":"2024-09-21T10:16:50.940308+0000","last_became_active":"2024-09-21T10:14:45.530295+0000","last_became_peered":"2024-09-21T10:14:45.530295+0000","last_unstale":"2024-09-21T10:16:50.940308+0000","last_undegraded":"2024-09-21T10:16:50.940308+0000","last_fullsized":"2024-09-21T10:16:50.940308+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:34.920194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5071110168,"num_objects":77380,"num_object_clones":0,"num_object_copies":232140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10042,"num_objects_unfound":0,"num_objects_dirty":77380,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171948,"num_write_kb":7978433,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6135,"num_bytes_recovered":401997824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15532109824,"data_stored":15532097608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129348,"ondisk_log_size":129348,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":1370,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":164,"num_bytes_recovered":16777254,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234330,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2371700,"kb_used_data":1893484,"kb_used_omap":1,"kb_used_meta":478206,"kb_avail":91373452,"statfs":{"total":95995035648,"available":93566414848,"internally_reserved":0,"allocated":1938927616,"data_stored":1937628086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":489683403},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495968,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2310008,"kb_used_data":1881840,"kb_used_omap":1,"kb_used_meta":428158,"kb_avail":91435144,"statfs":{"total":95995035648,"available":93629587456,"internally_reserved":0,"allocated":1927004160,"data_stored":1925701344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":438434247},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757605,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1962468,"kb_used_data":1588444,"kb_used_omap":1,"kb_used_meta":374014,"kb_avail":91782684,"statfs":{"total":95995035648,"available":93985468416,"internally_reserved":0,"allocated":1626566656,"data_stored":1625278996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":382990790},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019241,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2301808,"kb_used_data":1883496,"kb_used_omap":1,"kb_used_meta":418302,"kb_avail":91443344,"statfs":{"total":95995035648,"available":93637984256,"internally_reserved":0,"allocated":1928699904,"data_stored":1927404187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":428341702},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435426,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3902464,"kb_used_data":3361200,"kb_used_omap":5,"kb_used_meta":541178,"kb_avail":89842688,"statfs":{"total":95995035648,"available":91998912512,"internally_reserved":0,"allocated":3441868800,"data_stored":3440576920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":554166312},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598953,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2354804,"kb_used_data":1821932,"kb_used_omap":7,"kb_used_meta":532856,"kb_avail":91390348,"statfs":{"total":95995035648,"available":93583716352,"internally_reserved":0,"allocated":1865658368,"data_stored":1864370778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":545645005},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762479,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2911972,"kb_used_data":2515164,"kb_used_omap":7,"kb_used_meta":396792,"kb_avail":90833180,"statfs":{"total":95995035648,"available":93013176320,"internally_reserved":0,"allocated":2575527936,"data_stored":2574239953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":406315261},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926006,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":636184,"kb_used_data":311568,"kb_used_omap":7,"kb_used_meta":324600,"kb_avail":93108968,"statfs":{"total":95995035648,"available":95343583232,"internally_reserved":0,"allocated":319045632,"data_stored":317769601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":332390590},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":316604416,"data_stored":316604416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2551123968,"data_stored":2551119896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1858338816,"data_stored":1858338816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3429830656,"data_stored":3429826584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1920073728,"data_stored":1920073728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1615003648,"data_stored":1615003648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1921585152,"data_stored":1921581080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1919549440,"data_stored":1919549440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:17:17.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 1 2024-09-21T10:17:17.362 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:17.363 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.394 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:17.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:17.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:17.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:17.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:17.635 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.696 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:17.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:17.797 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:17.834 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:17.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:17.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:17.981 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:18.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.122 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:18.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:18.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:18 smithi032 ceph-mon[105748]: pgmap v820: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 8283/225654 objects misplaced (3.671%); 17 MiB/s, 276 objects/s recovering 2024-09-21T10:17:18.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:18 smithi032 ceph-mon[101299]: pgmap v820: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 8283/225654 objects misplaced (3.671%); 17 MiB/s, 276 objects/s recovering 2024-09-21T10:17:18.223 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:18.244 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:18 smithi045 ceph-mon[95976]: pgmap v820: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 8283/225654 objects misplaced (3.671%); 17 MiB/s, 276 objects/s recovering 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:18.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:18.325 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:17:18.325 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:18.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:18.365 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:17:18.366 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:18.400 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:18.405 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:17:18.406 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:18.439 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:17:18.439 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:18.479 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:17:18.479 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:18.502 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:18.541 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:18.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:18.643 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:18.681 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:18.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:18.783 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:18.787 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:17:18.787 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:17:18.787 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.787 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:177780 7.6 7:6a41d89a:::benchmark_data_smithi032_200324_object53114:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-21T10:17:18.787 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:18.778439+0000", 2024-09-21T10:17:18.787 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0055284940000000001, 2024-09-21T10:17:18.787 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0055563189999999997, 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 177780 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778439+0000", 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:18.788 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778439+0000", 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778438+0000", 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778444+0000", 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5010000000000001e-06 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778460+0000", 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5645e-05 2024-09-21T10:17:18.789 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778467+0000", 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5259999999999996e-06 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782223+0000", 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0037555000000000002 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782343+0000", 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012000199999999999 2024-09-21T10:17:18.790 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3,5", 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782479+0000", 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013661200000000001 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:177773 7.8 e278/245 7:1cee0b5d:::benchmark_data_smithi032_200324_object53107:head v 278'11056, mlcod=278'11052)", 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:18.778484+0000", 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0054834319999999999, 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.005589536, 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:17:18.791 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778484+0000", 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778484+0000", 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778484+0000", 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778489+0000", 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9099999999999996e-06 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778496+0000", 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7159999999999997e-06 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.778500+0000", 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4850000000000003e-06 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.793 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.780310+0000", 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0018095209999999999 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.780327+0000", 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7448999999999998e-05 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:177781 7.f e278/245 7:f3084bdc:::benchmark_data_smithi032_200324_object53115:head v 278'11176, mlcod=278'11174)", 2024-09-21T10:17:18.794 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:18.779664+0000", 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0043038019999999998, 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0044655900000000002, 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.779664+0000", 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.779664+0000", 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.795 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.779663+0000", 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.779674+0000", 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0902e-05 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.779685+0000", 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0740000000000001e-05 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.796 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.779691+0000", 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1979999999999998e-06 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.780121+0000", 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00043040500000000002 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.780142+0000", 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0273999999999999e-05 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.797 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:177786 7.9 e278/248 7:90f823a0:::benchmark_data_smithi032_200324_object53120:head v 278'11131, mlcod=278'11129)", 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:18.782505+0000", 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.001463095, 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001678215, 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782505+0000", 2024-09-21T10:17:18.798 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782505+0000", 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782504+0000", 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782510+0000", 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6429999999999998e-06 2024-09-21T10:17:18.799 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782520+0000", 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3710000000000006e-06 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782526+0000", 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1840000000000004e-06 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.783576+0000", 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0010503279999999999 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.783596+0000", 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9910000000000001e-05 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45206.0:177770 7.8 e278/245 7:16b4a552:::benchmark_data_smithi032_200324_object53104:head v 278'11055, mlcod=278'11052)", 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:18.774623+0000", 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0093445799999999999, 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0096156169999999999, 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.801 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774623+0000", 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774623+0000", 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774622+0000", 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.802 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774628+0000", 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4689999999999996e-06 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774636+0000", 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2060000000000007e-06 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774642+0000", 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4280000000000004e-06 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.803 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774783+0000", 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014176799999999999 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774807+0000", 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3326e-05 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:177777 7.6 7:654dae36:::benchmark_data_smithi032_200324_object53111:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-21T10:17:18.804 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:18.774662+0000", 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0093061499999999991, 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0096336519999999995, 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 177777 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774662+0000", 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.805 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774662+0000", 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774660+0000", 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774665+0000", 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9470000000000002e-06 2024-09-21T10:17:18.806 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774675+0000", 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0013000000000001e-05 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.774682+0000", 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4300000000000003e-06 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.777185+0000", 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0025032349999999999 2024-09-21T10:17:18.807 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.777300+0000", 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011534700000000001 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3,5", 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.777441+0000", 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014037300000000001 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:18.782190+0000", 2024-09-21T10:17:18.808 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00474896 2024-09-21T10:17:18.809 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.809 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:18.809 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.809 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:18.809 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:17:18.809 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 6 2024-09-21T10:17:18.809 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:17:18.822 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:18.924 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.959 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:18.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:19.060 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.098 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:19.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:19.199 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:19.245 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:19.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:19.346 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:19.364 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4118065093' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:19.364 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:19.364 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4118065093' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:19.365 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:19.378 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:19.480 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:19.519 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4118065093' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:19.595 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:17:19.621 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:19.636 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:17:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:17:19] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-21T10:17:19.663 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:19.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:19.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:19.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.809 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:19.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:19.910 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:19.954 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:19.954 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:20.050 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:20.056 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:20.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.115 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:20.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:20.216 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:20.235 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:20 smithi032 ceph-mon[101299]: pgmap v821: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 8283/222708 objects misplaced (3.719%); 10 MiB/s, 162 objects/s recovering 2024-09-21T10:17:20.236 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:20 smithi032 ceph-mon[105748]: pgmap v821: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 8283/222708 objects misplaced (3.719%); 10 MiB/s, 162 objects/s recovering 2024-09-21T10:17:20.269 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:20.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:20.371 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:20.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:20.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:20.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:20 smithi045 ceph-mon[95976]: pgmap v821: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 8283/222708 objects misplaced (3.719%); 10 MiB/s, 162 objects/s recovering 2024-09-21T10:17:20.554 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:20.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:20.656 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:20.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:20.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:20.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.844 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:20.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:20.945 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:20.967 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:21.069 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:21.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:21.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:21.208 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:21.250 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:21.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:21.352 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:21.378 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:21.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:21.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:17:21.481 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:21.537 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:21.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:21.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:21.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.684 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:21.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:21.785 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:21.818 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:21.920 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:21.950 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:22.053 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:22.085 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:22.178 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:22 smithi032 ceph-mon[101299]: pgmap v822: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 7116/213438 objects misplaced (3.334%); 14 MiB/s, 228 objects/s recovering 2024-09-21T10:17:22.178 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:22 smithi032 ceph-mon[105748]: pgmap v822: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 7116/213438 objects misplaced (3.334%); 14 MiB/s, 228 objects/s recovering 2024-09-21T10:17:22.184 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:22 smithi045 ceph-mon[95976]: pgmap v822: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 7116/213438 objects misplaced (3.334%); 14 MiB/s, 228 objects/s recovering 2024-09-21T10:17:22.187 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:22.238 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:22.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:22.344 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:22.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.388 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:22.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:22.489 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:22.507 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:17:22.508 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:17:22.508 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:17:22.508 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:17:22.508 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:17:22.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.531 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:22.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:22.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:22.662 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:22.699 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:22.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:22.808 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:22.910 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:22.941 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:22.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:23.043 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:23.070 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:17:23.091 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:23.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1804385243' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:23.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:23 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:23.192 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:23.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1804385243' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:23.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:23 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:23.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1804385243' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:23.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:23 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:23.220 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:17:23.243 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:23.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:23.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:23.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:23.481 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:23.520 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:23.552 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:23.622 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:23.663 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:23.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:23.764 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.797 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:23.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:23.898 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:23.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.931 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:23.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:24.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:24.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.074 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:24.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:24.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:24.204 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:24.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:24 smithi045 ceph-mon[95976]: pgmap v823: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 7116/213438 objects misplaced (3.334%); 13 MiB/s, 200 objects/s recovering 2024-09-21T10:17:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:24 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:17:24.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:24 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:17:24.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:24 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:24.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:24 smithi045 ceph-mon[95976]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T10:17:24.306 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:24.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:24.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:24.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[105748]: pgmap v823: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 7116/213438 objects misplaced (3.334%); 13 MiB/s, 200 objects/s recovering 2024-09-21T10:17:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:17:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:17:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[105748]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T10:17:24.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[101299]: pgmap v823: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 7116/213438 objects misplaced (3.334%); 13 MiB/s, 200 objects/s recovering 2024-09-21T10:17:24.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:17:24.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:17:24.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:24.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:24 smithi032 ceph-mon[101299]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T10:17:24.469 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:24.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:24.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:24.614 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:24.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:24.716 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:24.754 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:24.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:24.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.915 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:24.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:25.016 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:25.042 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:25.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:25.106 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 1 pgp_num to 1 2024-09-21T10:17:25.144 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:25.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.190 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:25.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:25.290 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:25.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:25.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:25.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/218798955' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:17:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:25 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:17:25.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/218798955' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:17:25.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:25 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:17:25.461 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/218798955' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:17:25.461 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:25 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-21T10:17:25.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:25.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:25.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:25.604 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:25.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:25.705 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:25.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:25.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213387 5.10 5.645680d0 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.883648+0000", 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.900628676, 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99656069300000005, 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.789 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213387 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883648+0000", 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.790 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883648+0000", 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883646+0000", 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883654+0000", 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2479999999999999e-06 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.791 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883669+0000", 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5434000000000002e-05 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883677+0000", 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6799999999999993e-06 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883759+0000", 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2169999999999994e-05 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.883767+0000", 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1629999999999996e-06 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880198+0000", 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99643097999999997 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.793 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880209+0000", 2024-09-21T10:17:25.794 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0875e-05 2024-09-21T10:17:25.794 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.794 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.794 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.794 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.794 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.794 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213398 5.10 5.bd5fbbf0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.794 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.892190+0000", 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.89208710800006, 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98813121900000001, 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213398 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892190+0000", 2024-09-21T10:17:25.795 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892190+0000", 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892189+0000", 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892194+0000", 2024-09-21T10:17:25.796 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9169999999999997e-06 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892205+0000", 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1073000000000001e-05 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892213+0000", 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9379999999999992e-06 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.797 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892273+0000", 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0674999999999998e-05 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.892284+0000", 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0335e-05 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880308+0000", 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98802432900000003 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.798 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880321+0000", 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3022e-05 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213409 5.10 5.e3d5e190 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.899439+0000", 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.884837683, 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98091395400000003, 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.799 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213409 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899439+0000", 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899439+0000", 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.800 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899438+0000", 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899443+0000", 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8749999999999999e-06 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899454+0000", 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1017e-05 2024-09-21T10:17:25.801 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899461+0000", 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.391e-06 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899513+0000", 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2345000000000001e-05 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.899522+0000", 2024-09-21T10:17:25.802 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0590000000000003e-06 2024-09-21T10:17:25.803 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.803 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.803 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.803 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880345+0000", 2024-09-21T10:17:25.803 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98082366899999995 2024-09-21T10:17:25.803 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.803 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.803 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.803 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880353+0000", 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0569999999999993e-06 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213416 5.10 5.10f69050 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.905431+0000", 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.87884568200002, 2024-09-21T10:17:25.804 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97496326499999997, 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213416 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905431+0000", 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.805 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905431+0000", 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905430+0000", 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905438+0000", 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0849999999999997e-06 2024-09-21T10:17:25.806 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905450+0000", 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1790000000000001e-05 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905456+0000", 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.19e-06 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905491+0000", 2024-09-21T10:17:25.807 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4449999999999997e-05 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.905495+0000", 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6099999999999999e-06 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880383+0000", 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97488764400000005 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880395+0000", 2024-09-21T10:17:25.808 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.154e-05 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213453 5.10 5.4c1650d0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.939270+0000", 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.84500664100005, 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.94115652100000002, 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.809 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213453 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939270+0000", 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939270+0000", 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.810 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939269+0000", 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939275+0000", 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7909999999999999e-06 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939291+0000", 2024-09-21T10:17:25.811 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5420000000000001e-05 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939296+0000", 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3290000000000001e-06 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939341+0000", 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4842000000000002e-05 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.939350+0000", 2024-09-21T10:17:25.812 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8880000000000003e-06 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880416+0000", 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.94106558200000001 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880427+0000", 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1365e-05 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.813 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213458 5.10 5.aa724650 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.947131+0000", 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.83714578499996, 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.93332198799999999, 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213458 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.814 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947131+0000", 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947131+0000", 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947130+0000", 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.815 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947136+0000", 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8250000000000001e-06 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947145+0000", 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2369999999999998e-06 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947152+0000", 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.64e-06 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.816 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947195+0000", 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2960000000000002e-05 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.947203+0000", 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2269999999999993e-06 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880444+0000", 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.93324136300000005 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.817 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880453+0000", 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8200000000000003e-06 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213507 5.10 5.fbac80f0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:32.985956+0000", 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.79832129600004, 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.89453823399999999, 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.818 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213507 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985956+0000", 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.819 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985956+0000", 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985955+0000", 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985960+0000", 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3979999999999998e-06 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.820 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985975+0000", 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4622e-05 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.985983+0000", 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9179999999999994e-06 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.986035+0000", 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2018999999999998e-05 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.821 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:32.986042+0000", 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4869999999999996e-06 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880485+0000", 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.89444252199999996 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880494+0000", 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3680000000000008e-06 2024-09-21T10:17:25.822 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213582 5.10 5.ab58c590 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.048306+0000", 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.73597083699997, 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.83221718, 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213582 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048306+0000", 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048306+0000", 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.824 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048305+0000", 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048310+0000", 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9150000000000004e-06 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048321+0000", 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0861e-05 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.825 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048327+0000", 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6370000000000002e-06 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048387+0000", 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9551e-05 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.048394+0000", 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.277e-06 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.826 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880518+0000", 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.83212388800000003 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880523+0000", 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2519999999999999e-06 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213626 5.10 5.4cd189b0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.084187+0000", 2024-09-21T10:17:25.827 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.70008988400002, 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.79637013700000003, 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213626 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084187+0000", 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.828 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084187+0000", 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084186+0000", 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084191+0000", 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4729999999999999e-06 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084202+0000", 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0596e-05 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084209+0000", 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0360000000000001e-06 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084256+0000", 2024-09-21T10:17:25.830 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7663000000000001e-05 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.084264+0000", 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0220000000000004e-06 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880549+0000", 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.79628444099999995 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.831 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.832 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.832 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880557+0000", 2024-09-21T10:17:25.832 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3350000000000007e-06 2024-09-21T10:17:25.832 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.832 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.832 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.832 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.832 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.832 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213633 5.10 5.18cc6f50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.092236+0000", 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.69204111800002, 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.78834528699999995, 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213633 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.833 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092236+0000", 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092236+0000", 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092234+0000", 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:17:25.834 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092243+0000", 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4630000000000007e-06 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092259+0000", 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5245000000000001e-05 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.835 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092267+0000", 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6300000000000004e-06 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092311+0000", 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3951e-05 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.092319+0000", 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7640000000000005e-06 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.836 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880576+0000", 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.78825735200000002 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880581+0000", 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7190000000000001e-06 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213651 5.10 5.b09713b0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.108041+0000", 2024-09-21T10:17:25.837 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.67623567400005, 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.77257185100000003, 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213651 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108041+0000", 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.838 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108041+0000", 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108040+0000", 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108045+0000", 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4750000000000004e-06 2024-09-21T10:17:25.839 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108074+0000", 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9459999999999999e-05 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108086+0000", 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2035000000000001e-05 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108144+0000", 2024-09-21T10:17:25.840 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8010000000000002e-05 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.108153+0000", 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0830000000000004e-06 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880605+0000", 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.772452162 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880613+0000", 2024-09-21T10:17:25.841 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4419999999999998e-06 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213671 5.10 5.775f8bb0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.127502+0000", 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.656775444, 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.75312580900000003, 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.842 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213671 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127502+0000", 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127502+0000", 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.843 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127501+0000", 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127505+0000", 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6770000000000003e-06 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127516+0000", 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0674e-05 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.844 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127521+0000", 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8060000000000002e-06 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127557+0000", 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6684999999999999e-05 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.127562+0000", 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9509999999999997e-06 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.845 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880623+0000", 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.75306079199999998 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880627+0000", 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.245e-06 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213799 5.10 5.20d823f0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.846 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.279962+0000", 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.504314583, 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.60068750100000001, 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213799 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279962+0000", 2024-09-21T10:17:25.847 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279962+0000", 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279961+0000", 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279967+0000", 2024-09-21T10:17:25.848 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.665e-06 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279978+0000", 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1405e-05 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.279985+0000", 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.652e-06 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.849 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.280054+0000", 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8783999999999997e-05 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.280062+0000", 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9780000000000005e-06 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880642+0000", 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.60057990299999997 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.850 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880650+0000", 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2979999999999992e-06 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213835 5.10 5.b1c1bcf0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.339730+0000", 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.44454669900006, 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54093409000000003, 2024-09-21T10:17:25.851 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213835 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.852 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339730+0000", 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339730+0000", 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339729+0000", 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.853 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.854 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339736+0000", 2024-09-21T10:17:25.854 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9240000000000001e-06 2024-09-21T10:17:25.854 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.854 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.854 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.854 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339757+0000", 2024-09-21T10:17:25.854 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1168000000000001e-05 2024-09-21T10:17:25.855 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.855 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.855 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.855 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339766+0000", 2024-09-21T10:17:25.855 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7649999999999999e-06 2024-09-21T10:17:25.855 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.856 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.856 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.856 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339824+0000", 2024-09-21T10:17:25.856 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7893000000000003e-05 2024-09-21T10:17:25.856 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.856 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.856 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.856 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.339833+0000", 2024-09-21T10:17:25.857 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.628e-06 2024-09-21T10:17:25.857 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.857 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.857 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.857 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880660+0000", 2024-09-21T10:17:25.857 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54082699199999995 2024-09-21T10:17:25.857 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.857 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.857 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.858 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880664+0000", 2024-09-21T10:17:25.858 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.233e-06 2024-09-21T10:17:25.858 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.858 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.858 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.858 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.858 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.859 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213850 5.10 5.b9b1c390 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.859 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.365015+0000", 2024-09-21T10:17:25.859 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.41926155399995, 2024-09-21T10:17:25.859 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.515670454, 2024-09-21T10:17:25.859 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.859 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.859 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.860 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.860 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.860 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213850 2024-09-21T10:17:25.860 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.860 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.860 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365015+0000", 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365015+0000", 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.861 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365014+0000", 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365021+0000", 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6640000000000001e-06 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365033+0000", 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2131e-05 2024-09-21T10:17:25.862 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365038+0000", 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5319999999999997e-06 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365082+0000", 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3278999999999997e-05 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.863 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.864 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.864 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.365090+0000", 2024-09-21T10:17:25.864 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2150000000000001e-06 2024-09-21T10:17:25.864 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.864 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.864 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.864 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880677+0000", 2024-09-21T10:17:25.864 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:25.865 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.51558764300000004 2024-09-21T10:17:25.865 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.865 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.865 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880686+0000", 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4400000000000005e-06 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.44543.0:213854 5.10 5.6b4dc610 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:08:33.376078+0000", 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.40819928300004, 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.50462188200000002, 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.866 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.44543", 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 213854 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376078+0000", 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376078+0000", 2024-09-21T10:17:25.867 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376077+0000", 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376081+0000", 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2540000000000003e-06 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376090+0000", 2024-09-21T10:17:25.868 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0000000000000002e-06 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376094+0000", 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.98e-06 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376130+0000", 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6358e-05 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.869 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.376136+0000", 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6339999999999998e-06 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880695+0000", 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.50455955799999996 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:08:33.880700+0000", 2024-09-21T10:17:25.870 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1620000000000001e-06 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:2401 7.6 7:62a2a95e:::benchmark_data_smithi032_200324_object2400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:14:44.520838+0000", 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 161.26343874, 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2822637779999999, 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:17:25.871 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 2401 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.520838+0000", 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.520838+0000", 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.872 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.520837+0000", 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.520865+0000", 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7977999999999999e-05 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.873 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.520877+0000", 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1800999999999999e-05 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.520885+0000", 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2509999999999994e-06 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.520927+0000", 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2020000000000001e-05 2024-09-21T10:17:25.874 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.521069+0000", 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014170200000000001 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.521227+0000", 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000157873 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.875 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.521744+0000", 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00051735799999999999 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.523766+0000", 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0020214849999999999 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.876 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.526053+0000", 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022875399999999998 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.526066+0000", 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3152e-05 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.877 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:45.523714+0000", 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99764790199999998 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:45.523730+0000", 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5238999999999999e-05 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.799126+0000", 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2753965570000001 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.878 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for degraded object", 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.799817+0000", 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00069068000000000005 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.803007+0000", 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031903330000000001 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.803102+0000", 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4924000000000001e-05 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.879 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:2411 7.6 7:658f2b61:::benchmark_data_smithi032_200324_object2410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:14:44.527319+0000", 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 161.25695825099999, 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.278368258, 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 2411 2024-09-21T10:17:25.880 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527319+0000", 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527319+0000", 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.881 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527317+0000", 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527351+0000", 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3683000000000003e-05 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527363+0000", 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2551999999999999e-05 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.882 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527371+0000", 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6799999999999993e-06 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527486+0000", 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115102 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.883 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527496+0000", 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.7929999999999992e-06 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:45.523763+0000", 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99626695899999995 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:45.523771+0000", 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4060000000000005e-06 2024-09-21T10:17:25.884 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.800180+0000", 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.276408964 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.800439+0000", 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00025907099999999999 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.800746+0000", 2024-09-21T10:17:25.885 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00030642899999999999 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.802207+0000", 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001461176 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.802247+0000", 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0122e-05 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.886 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.805464+0000", 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0032175070000000001 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.805506+0000", 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1820999999999998e-05 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.887 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.805687+0000", 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000180762 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:2412 7.6 7:6d6e0e4b:::benchmark_data_smithi032_200324_object2411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:14:44.527394+0000", 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 161.25688312, 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2784738899999999, 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:25.888 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 2412 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527394+0000", 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527394+0000", 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.889 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527393+0000", 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527423+0000", 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9725000000000001e-05 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.890 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527433+0000", 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.7419999999999992e-06 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527439+0000", 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2580000000000001e-06 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527529+0000", 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0381000000000001e-05 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.891 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:44.527538+0000", 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.524e-06 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:45.523979+0000", 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99644153099999999 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:45.523989+0000", 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0850000000000003e-06 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.892 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.801245+0000", 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2772566830000001 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.801391+0000", 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000145458 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.801571+0000", 2024-09-21T10:17:25.893 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00018068899999999999 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.802272+0000", 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00070025999999999997 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.803206+0000", 2024-09-21T10:17:25.894 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00093404299999999996 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.805728+0000", 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0025227549999999998 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.805764+0000", 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5645999999999997e-05 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.895 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:14:46.805868+0000", 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000103698 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:157447 7.6 7:6e7663e6:::benchmark_data_smithi032_200324_object32781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:16:52.368865+0000", 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 33.415411777999999, 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.17816996299999999, 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:17:25.896 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 157447 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.368865+0000", 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.368865+0000", 2024-09-21T10:17:25.897 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.368864+0000", 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.368870+0000", 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5119999999999997e-06 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:17:25.898 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.368882+0000", 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2069000000000001e-05 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.368889+0000", 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6959999999999999e-06 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.383045+0000", 2024-09-21T10:17:25.899 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014155511000000001 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.383207+0000", 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016214499999999999 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3,5", 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.383398+0000", 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000191695 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:25.900 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.388121+0000", 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0047220430000000004 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.392665+0000", 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0045440580000000001 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.395803+0000", 2024-09-21T10:17:25.901 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031388369999999998 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.546882+0000", 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15107831199999999 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.546931+0000", 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8835000000000003e-05 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:17:25.902 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:16:52.547035+0000", 2024-09-21T10:17:25.903 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000104609 2024-09-21T10:17:25.903 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.903 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.903 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.903 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:17:25.903 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:17:25.903 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:17:25.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.936 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:25.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:26.037 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.072 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:26.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:26.172 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:26.205 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:26.307 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:26.344 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:26.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:26.445 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:26 smithi032 ceph-mon[105748]: pgmap v825: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 7116/208725 objects misplaced (3.409%); 13 MiB/s, 200 objects/s recovering 2024-09-21T10:17:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:26 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:17:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:26 smithi032 ceph-mon[105748]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T10:17:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:26 smithi032 ceph-mon[101299]: pgmap v825: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 7116/208725 objects misplaced (3.409%); 13 MiB/s, 200 objects/s recovering 2024-09-21T10:17:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:26 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:17:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:26 smithi032 ceph-mon[101299]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T10:17:26.479 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:26 smithi045 ceph-mon[95976]: pgmap v825: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 7116/208725 objects misplaced (3.409%); 13 MiB/s, 200 objects/s recovering 2024-09-21T10:17:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:26 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-21T10:17:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:26 smithi045 ceph-mon[95976]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T10:17:26.581 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:26.617 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:26.719 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:26.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:26.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:26.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:26.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.886 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:26.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:26.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:27.026 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:27.128 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:27.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:27.269 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:27.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.299 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:27.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:27.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:27.435 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:27.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:27.537 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:27.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.573 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:27.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:27.658 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:17:27.674 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.755 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:27.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:27.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:27.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:28.006 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:28.041 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:28.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:28.142 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:28.176 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:28.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:28.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:28.329 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:28.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:28.356 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:28 smithi032 ceph-mon[101299]: pgmap v827: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 5916/201021 objects misplaced (2.943%); 12 MiB/s, 192 objects/s recovering 2024-09-21T10:17:28.356 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:28 smithi032 ceph-mon[105748]: pgmap v827: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 5916/201021 objects misplaced (2.943%); 12 MiB/s, 192 objects/s recovering 2024-09-21T10:17:28.432 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:28.462 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:28 smithi045 ceph-mon[95976]: pgmap v827: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 5916/201021 objects misplaced (2.943%); 12 MiB/s, 192 objects/s recovering 2024-09-21T10:17:28.565 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:28.622 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:28.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:28.724 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:28.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:28.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:28.844 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:17:28.844 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:28.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:28.863 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:28.901 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:17:28.901 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:28.931 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:28.944 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:17:28.944 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:28.994 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:17:28.995 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:29.025 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:17:29.025 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:29.033 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:29.063 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:17:29.063 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.076 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:29.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:29.099 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:17:29.099 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:29.137 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:17:29.138 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:29.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:29.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:29.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:29.325 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:29.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.363 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:29.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:29.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:29 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4057705539' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:29.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:29 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4057705539' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:29.464 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:29.505 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:29.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:29 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4057705539' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:29.607 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:29.643 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:29.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:29.745 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:29.785 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:29.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:29.886 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:29.919 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:29.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:17:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:17:29] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-21T10:17:29.971 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:17:30.022 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:30.054 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:30.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:30.156 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:30.201 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:30.273 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:30 smithi032 ceph-mon[105748]: pgmap v828: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 5916/198027 objects misplaced (2.987%); 6.5 MiB/s, 104 objects/s recovering 2024-09-21T10:17:30.273 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:30 smithi032 ceph-mon[101299]: pgmap v828: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 5916/198027 objects misplaced (2.987%); 6.5 MiB/s, 104 objects/s recovering 2024-09-21T10:17:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:30 smithi045 ceph-mon[95976]: pgmap v828: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 5916/198027 objects misplaced (2.987%); 6.5 MiB/s, 104 objects/s recovering 2024-09-21T10:17:30.303 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:30.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:30.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:30.469 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:30.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:30.571 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:30.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:30.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:30.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:30.746 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:30.848 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:30.886 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:30.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:30.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:31.023 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:31.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:31.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:31.147 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:31.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:31.249 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:31.290 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:31.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:31.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:31.423 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:31.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:31.525 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.566 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:31.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:31.667 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:31.692 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:31.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:31.794 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:31.827 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:31.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:31.929 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:31.961 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:31.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:32.062 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:32.109 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:32.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:32.159 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:17:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:32 smithi032 ceph-mon[101299]: pgmap v829: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 4851/189162 objects misplaced (2.564%); 12 MiB/s, 196 objects/s recovering 2024-09-21T10:17:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:32 smithi032 ceph-mon[105748]: pgmap v829: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 4851/189162 objects misplaced (2.564%); 12 MiB/s, 196 objects/s recovering 2024-09-21T10:17:32.211 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:32.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.275 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:32.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:32.376 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:32.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.433 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:32.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:32.534 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:32 smithi045 ceph-mon[95976]: pgmap v829: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 4851/189162 objects misplaced (2.564%); 12 MiB/s, 196 objects/s recovering 2024-09-21T10:17:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.577 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:32.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:32.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:32.704 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:32.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:32.806 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:32.823 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:32.833 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:32.935 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.978 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:32.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:33.026 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:33.079 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:33.105 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:33.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:33.165 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:17:33.165 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:17:33.165 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:17:33.166 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T10:17:33.166 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.5 2024-09-21T10:17:33.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:33.249 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:33.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:33.351 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:33.390 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:33.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:33.492 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:33.529 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:33.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:33.535 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:33 smithi045 systemd[1]: Stopping Ceph osd.5 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:17:33.630 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:33.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:33.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:33.767 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:33.801 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:33.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:33.903 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:33.940 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:34.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:34.080 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:34.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:34.124 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:17:34.134 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[43716]: 2024-09-21T10:17:33.786+0000 7f9bb73db640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:17:34.134 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[43716]: 2024-09-21T10:17:33.786+0000 7f9bb73db640 -1 osd.5 280 *** Got signal Terminated *** 2024-09-21T10:17:34.134 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[43716]: 2024-09-21T10:17:33.786+0000 7f9bb73db640 -1 osd.5 280 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:17:34.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:34.206 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:34.307 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[105748]: pgmap v830: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 4851/189162 objects misplaced (2.564%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1319537291' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[105748]: osd.5 marked itself down and dead 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[101299]: pgmap v830: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 4851/189162 objects misplaced (2.564%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1319537291' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:34.319 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:34 smithi032 ceph-mon[101299]: osd.5 marked itself down and dead 2024-09-21T10:17:34.330 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:34.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:34.432 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:34.456 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:34 smithi045 ceph-mon[95976]: pgmap v830: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 4851/189162 objects misplaced (2.564%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:17:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1319537291' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:17:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:34 smithi045 ceph-mon[95976]: osd.5 marked itself down and dead 2024-09-21T10:17:34.557 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:34.580 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:34.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:34.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:34.706 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:34.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:34.808 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:34.831 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:34.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:34.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:34.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:35.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:34 smithi045 podman[159928]: 2024-09-21 10:17:34.590866431 +0000 UTC m=+1.114918257 container died 490d4caa0702ac33f23f30fbb37f7a9c62faf3a2f40173a26b077ceb43ab1fe1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T10:17:35.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:35.081 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:35.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:35.183 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:35.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:35.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:35.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:35.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.356 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:35.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:35.456 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[101299]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1319537291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[101299]: osdmap e281: 8 total, 7 up, 8 in 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[101299]: pgmap v832: 89 pgs: 12 stale+active+clean, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 75 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 4851/184881 objects misplaced (2.624%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[101299]: osdmap e282: 8 total, 7 up, 8 in 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[105748]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1319537291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[105748]: osdmap e281: 8 total, 7 up, 8 in 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[105748]: pgmap v832: 89 pgs: 12 stale+active+clean, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 75 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 4851/184881 objects misplaced (2.624%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:17:35.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:35 smithi032 ceph-mon[105748]: osdmap e282: 8 total, 7 up, 8 in 2024-09-21T10:17:35.464 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:35 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:35.464 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:35 smithi045 ceph-mon[95976]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:17:35.464 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:35 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1319537291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:17:35.464 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:35 smithi045 ceph-mon[95976]: osdmap e281: 8 total, 7 up, 8 in 2024-09-21T10:17:35.464 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:35 smithi045 ceph-mon[95976]: pgmap v832: 89 pgs: 12 stale+active+clean, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 75 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 4851/184881 objects misplaced (2.624%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:17:35.464 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:35 smithi045 ceph-mon[95976]: osdmap e282: 8 total, 7 up, 8 in 2024-09-21T10:17:35.490 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:35.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:35.591 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:35.625 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:35.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:35.727 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:35.770 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:35.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:35.785 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:35 smithi045 podman[159928]: 2024-09-21 10:17:35.568965747 +0000 UTC m=+2.093017570 container cleanup 490d4caa0702ac33f23f30fbb37f7a9c62faf3a2f40173a26b077ceb43ab1fe1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:17:35.785 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:35 smithi045 bash[159928]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5 2024-09-21T10:17:35.871 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:35.909 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:36.011 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:36.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.052 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:36.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:36.152 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:36.190 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:36.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:36 smithi045 ceph-mon[95976]: 7.6 continuing backfill to osd.0 from (263'4300,280'12013] 7:6f526bfd:::benchmark_data_smithi032_200324_object88503:head to 280'12013 2024-09-21T10:17:36.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:35 smithi045 podman[159994]: 2024-09-21 10:17:35.912533527 +0000 UTC m=+1.317272557 container remove 490d4caa0702ac33f23f30fbb37f7a9c62faf3a2f40173a26b077ceb43ab1fe1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:17:36.291 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:36.318 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:36.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:36.420 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:36.435 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:36 smithi032 ceph-mon[105748]: 7.6 continuing backfill to osd.0 from (263'4300,280'12013] 7:6f526bfd:::benchmark_data_smithi032_200324_object88503:head to 280'12013 2024-09-21T10:17:36.435 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:36 smithi032 ceph-mon[101299]: 7.6 continuing backfill to osd.0 from (263'4300,280'12013] 7:6f526bfd:::benchmark_data_smithi032_200324_object88503:head to 280'12013 2024-09-21T10:17:36.445 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:36.547 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:36.571 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:36.672 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:36.695 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:36.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:36.785 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:36 smithi045 podman[160156]: 2024-09-21 10:17:36.302941757 +0000 UTC m=+0.026472688 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:17:36.785 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:36 smithi045 podman[160156]: 2024-09-21 10:17:36.430397736 +0000 UTC m=+0.153928646 container create cc9e4feebf4496c25702163afecdec2dbd225791fe0bd6c675d9de04da17c366 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:17:36.797 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:36.819 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:36.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:36.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:36.944 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:36.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:37.046 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:37.069 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:37.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:37.159 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:36 smithi045 podman[160156]: 2024-09-21 10:17:36.891713107 +0000 UTC m=+0.615244020 container init cc9e4feebf4496c25702163afecdec2dbd225791fe0bd6c675d9de04da17c366 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:17:37.159 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:36 smithi045 podman[160156]: 2024-09-21 10:17:36.896012173 +0000 UTC m=+0.619543087 container start cc9e4feebf4496c25702163afecdec2dbd225791fe0bd6c675d9de04da17c366 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T10:17:37.171 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:37.194 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:37 smithi032 ceph-mon[101299]: osdmap e283: 8 total, 7 up, 8 in 2024-09-21T10:17:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:37 smithi032 ceph-mon[101299]: pgmap v835: 89 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 16 peering, 4 stale+active+clean, 67 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 3216/178905 objects misplaced (1.798%); 13 MiB/s, 205 objects/s recovering 2024-09-21T10:17:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:37 smithi032 ceph-mon[105748]: osdmap e283: 8 total, 7 up, 8 in 2024-09-21T10:17:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:37 smithi032 ceph-mon[105748]: pgmap v835: 89 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 16 peering, 4 stale+active+clean, 67 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 3216/178905 objects misplaced (1.798%); 13 MiB/s, 205 objects/s recovering 2024-09-21T10:17:37.296 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:37.343 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:37.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:37.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:37.491 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:37.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:37 smithi045 ceph-mon[95976]: osdmap e283: 8 total, 7 up, 8 in 2024-09-21T10:17:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:37 smithi045 ceph-mon[95976]: pgmap v835: 89 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 16 peering, 4 stale+active+clean, 67 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 3216/178905 objects misplaced (1.798%); 13 MiB/s, 205 objects/s recovering 2024-09-21T10:17:37.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:37.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:37.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:37.774 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:37.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:37.876 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.927 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:37.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:38.008 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:37 smithi045 podman[160156]: 2024-09-21 10:17:37.627794139 +0000 UTC m=+1.351325062 container attach cc9e4feebf4496c25702163afecdec2dbd225791fe0bd6c675d9de04da17c366 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T10:17:38.008 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:17:37 smithi045 podman[160156]: 2024-09-21 10:17:37.628783504 +0000 UTC m=+1.352314425 container died cc9e4feebf4496c25702163afecdec2dbd225791fe0bd6c675d9de04da17c366 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:17:38.027 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:38.070 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:38.125 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:38.125 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:38.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:38.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:38.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:38 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T10:17:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:38 smithi045 ceph-mon[95976]: osdmap e284: 8 total, 7 up, 8 in 2024-09-21T10:17:38.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:17:38.322 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:38.342 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.5.service' 2024-09-21T10:17:38.398 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:38 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T10:17:38.398 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:38 smithi032 ceph-mon[101299]: osdmap e284: 8 total, 7 up, 8 in 2024-09-21T10:17:38.398 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:38 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T10:17:38.398 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:38 smithi032 ceph-mon[105748]: osdmap e284: 8 total, 7 up, 8 in 2024-09-21T10:17:38.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.414 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:38.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:38.417 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:17:38.417 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T10:17:38.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:38.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:38.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:38.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:38.688 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:38.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:38.825 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:38.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:38.927 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:38.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.972 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:38.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:39.073 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:39.113 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:39.215 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:39.254 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:39.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:39.295 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:17:39.295 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:17:39.295 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:17:39.295 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:39.296 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:39.296 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:39.296 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:17:39.296 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:17:39.296 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:17:39.296 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:17:39.296 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:39.296 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:39.297 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:17:39.297 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:39.297 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:39.356 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:39.385 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:39.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/349847299' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:39.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:39 smithi032 ceph-mon[101299]: pgmap v837: 89 pgs: 5 active+undersized, 1 active+remapped+backfilling, 17 peering, 2 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 3723/176718 objects degraded (2.107%); 3216/176718 objects misplaced (1.820%); 20 MiB/s, 324 objects/s recovering 2024-09-21T10:17:39.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/349847299' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:39.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:39 smithi032 ceph-mon[105748]: pgmap v837: 89 pgs: 5 active+undersized, 1 active+remapped+backfilling, 17 peering, 2 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 3723/176718 objects degraded (2.107%); 3216/176718 objects misplaced (1.820%); 20 MiB/s, 324 objects/s recovering 2024-09-21T10:17:39.487 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:39.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:39 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/349847299' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:39 smithi045 ceph-mon[95976]: pgmap v837: 89 pgs: 5 active+undersized, 1 active+remapped+backfilling, 17 peering, 2 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 3723/176718 objects degraded (2.107%); 3216/176718 objects misplaced (1.820%); 20 MiB/s, 324 objects/s recovering 2024-09-21T10:17:39.619 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:39.644 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:39.746 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:39.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:39.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:39.871 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:39.897 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:39.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:17:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:17:39] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-21T10:17:39.999 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:40.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:40.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:40.132 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:40.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:40.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:40.302 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:40.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:40.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:40.444 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:40.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:40.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:40 smithi032 ceph-mon[105748]: Health check failed: Degraded data redundancy: 3723/176718 objects degraded (2.107%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:17:40.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:40 smithi032 ceph-mon[101299]: Health check failed: Degraded data redundancy: 3723/176718 objects degraded (2.107%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:17:40.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:40 smithi045 ceph-mon[95976]: Health check failed: Degraded data redundancy: 3723/176718 objects degraded (2.107%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:17:40.546 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:40.572 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:40.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:40.674 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:40.710 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:40.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:40.812 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:40.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.845 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:40.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:40.946 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:40.973 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:17:40.974 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:17:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:40.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:41.075 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:41.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:41.210 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:41.255 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:41.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:41.356 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:41.403 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:41.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:41.452 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:41 smithi032 ceph-mon[101299]: pgmap v838: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 1 peering, 11 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 14401/170769 objects degraded (8.433%); 1468/170769 objects misplaced (0.860%); 32 MiB/s, 506 objects/s recovering 2024-09-21T10:17:41.452 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:41 smithi032 ceph-mon[105748]: pgmap v838: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 1 peering, 11 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 14401/170769 objects degraded (8.433%); 1468/170769 objects misplaced (0.860%); 32 MiB/s, 506 objects/s recovering 2024-09-21T10:17:41.505 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:41.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:41 smithi045 ceph-mon[95976]: pgmap v838: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 1 peering, 11 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 14401/170769 objects degraded (8.433%); 1468/170769 objects misplaced (0.860%); 32 MiB/s, 506 objects/s recovering 2024-09-21T10:17:41.546 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:41.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:41.647 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:41.675 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:41.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:41.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:41.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:41.880 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:17:41.906 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:41.906 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:41.930 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:41.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:42.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:42.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:42.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:42.201 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:42.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.263 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:42.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:42.364 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:42.390 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:42.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:42.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:42 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 7 pgs peering) 2024-09-21T10:17:42.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:42 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 7 pgs peering) 2024-09-21T10:17:42.491 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:42.519 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:42.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:42.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:42 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 7 pgs peering) 2024-09-21T10:17:42.620 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:42.663 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:42.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:42.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:42.796 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:42.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:42.897 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:42.937 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:42.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:42.974 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:17:43.039 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:43.066 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:43.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:43.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:43.207 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:43.309 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:43.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:43 smithi032 ceph-mon[101299]: pgmap v839: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 14401/170160 objects degraded (8.463%); 1468/170160 objects misplaced (0.863%); 15 MiB/s, 247 objects/s recovering 2024-09-21T10:17:43.325 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:43 smithi032 ceph-mon[105748]: pgmap v839: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 14401/170160 objects degraded (8.463%); 1468/170160 objects misplaced (0.863%); 15 MiB/s, 247 objects/s recovering 2024-09-21T10:17:43.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:43.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:43.417 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-09-21T10:17:43.417 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:17:43.418 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-21T10:17:43.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:17:43.456 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:43.479 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:43 smithi045 ceph-mon[95976]: pgmap v839: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 14401/170160 objects degraded (8.463%); 1468/170160 objects misplaced (0.863%); 15 MiB/s, 247 objects/s recovering 2024-09-21T10:17:43.544 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:43.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:43.646 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.713 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:43.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:43.814 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:43.893 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:43.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:43.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:44.023 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:44.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:44.125 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:44.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:44.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:44.255 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:44.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:44.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:44.385 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:44.423 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:44.524 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:44.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:44.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:44.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:44.695 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:44.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:44.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:44 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 13888/165486 objects degraded (8.392%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:44.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:44 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 13888/165486 objects degraded (8.392%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:44.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:44 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 13888/165486 objects degraded (8.392%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:44.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:44.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.841 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:44.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:44.942 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:44.943 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.988 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:44.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:45.089 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:45.115 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:45.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:45.217 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:45.242 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:45.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:45.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:45.369 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:45.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:45.471 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:45.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:45.534 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:17:45.613 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:45.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:45.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:45.758 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:45.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:45 smithi045 ceph-mon[95976]: pgmap v840: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 13888/165486 objects degraded (8.392%); 1468/165486 objects misplaced (0.887%); 14 MiB/s, 218 objects/s recovering 2024-09-21T10:17:45.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:45 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3974187399' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:45.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:45 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:45.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.788 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:45.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:45.889 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:45.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.923 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:45.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:45 smithi032 ceph-mon[105748]: pgmap v840: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 13888/165486 objects degraded (8.392%); 1468/165486 objects misplaced (0.887%); 14 MiB/s, 218 objects/s recovering 2024-09-21T10:17:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:45 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3974187399' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:45 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:45 smithi032 ceph-mon[101299]: pgmap v840: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 13888/165486 objects degraded (8.392%); 1468/165486 objects misplaced (0.887%); 14 MiB/s, 218 objects/s recovering 2024-09-21T10:17:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:45 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3974187399' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:45 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:46.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:46.063 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:46.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:46.164 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:46.205 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:46.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:46.307 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:46.352 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:46.454 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:46.490 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:46.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:46.592 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:46.623 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:46.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:46.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:46.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:46 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:46 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:46.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:46 smithi045 ceph-mon[95976]: osdmap e285: 8 total, 7 up, 8 in 2024-09-21T10:17:46.866 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:46.909 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:46.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:46 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:46 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:46 smithi032 ceph-mon[101299]: osdmap e285: 8 total, 7 up, 8 in 2024-09-21T10:17:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:46 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:46 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:46.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:46 smithi032 ceph-mon[105748]: osdmap e285: 8 total, 7 up, 8 in 2024-09-21T10:17:47.010 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:47.037 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:47.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:47.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:47.142 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.178 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:47.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:47.280 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:47.324 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:47.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:47.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:47.464 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:47.566 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:47.611 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:47.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:47 smithi032 ceph-mon[105748]: pgmap v842: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 13370/158373 objects degraded (8.442%); 515/158373 objects misplaced (0.325%); 17 MiB/s, 273 objects/s recovering 2024-09-21T10:17:47.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:47 smithi032 ceph-mon[101299]: pgmap v842: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 13370/158373 objects degraded (8.442%); 515/158373 objects misplaced (0.325%); 17 MiB/s, 273 objects/s recovering 2024-09-21T10:17:47.714 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:47.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:47.763 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:17:47.763 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:17:47.764 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:17:47.764 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:17:47.764 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:17:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:47 smithi045 ceph-mon[95976]: pgmap v842: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 13370/158373 objects degraded (8.442%); 515/158373 objects misplaced (0.325%); 17 MiB/s, 273 objects/s recovering 2024-09-21T10:17:47.857 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:47.901 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:47.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:48.003 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:48.037 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:48.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:48.139 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:48.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:48.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:48.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:48.304 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:48.406 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:48.450 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:48.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:48.552 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:48.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:48.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:48.691 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:48.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.714 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:48.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:48.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:48.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.839 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:48.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:48.940 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:48.981 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:48.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:49.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:49.107 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:49.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:49.209 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:49.232 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:49.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:49.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:49.356 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:49.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:49.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:49.480 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:49.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:49.607 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:49.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:49.655 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:17:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:17:49] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.51.0" 2024-09-21T10:17:49.656 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:49 smithi032 ceph-mon[101299]: pgmap v843: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 13100/154371 objects degraded (8.486%); 515/154371 objects misplaced (0.334%); 16 MiB/s, 252 objects/s recovering 2024-09-21T10:17:49.656 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:17:49.656 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:49.656 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:49 smithi032 ceph-mon[101299]: osdmap e286: 8 total, 7 up, 8 in 2024-09-21T10:17:49.656 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:49 smithi032 ceph-mon[105748]: pgmap v843: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 13100/154371 objects degraded (8.486%); 515/154371 objects misplaced (0.334%); 16 MiB/s, 252 objects/s recovering 2024-09-21T10:17:49.656 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:17:49.656 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:49.656 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:49 smithi032 ceph-mon[105748]: osdmap e286: 8 total, 7 up, 8 in 2024-09-21T10:17:49.710 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:49.736 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:49 smithi045 ceph-mon[95976]: pgmap v843: 89 pgs: 1 active+remapped+backfilling, 14 active+undersized, 11 active+undersized+degraded, 63 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 13100/154371 objects degraded (8.486%); 515/154371 objects misplaced (0.334%); 16 MiB/s, 252 objects/s recovering 2024-09-21T10:17:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:17:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:49 smithi045 ceph-mon[95976]: osdmap e286: 8 total, 7 up, 8 in 2024-09-21T10:17:49.838 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:49.866 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:49.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:49.967 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:49.996 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:49.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:50.098 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:50.139 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:50.240 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:50.272 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:50.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:50.373 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:50.413 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:50.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:50.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:50.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:50.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:50.646 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:50.684 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:50.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:50.702 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:50 smithi032 ceph-mon[105748]: osdmap e287: 8 total, 7 up, 8 in 2024-09-21T10:17:50.702 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:50 smithi032 ceph-mon[101299]: osdmap e287: 8 total, 7 up, 8 in 2024-09-21T10:17:50.786 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:50.827 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:50.929 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:50.958 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:50.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:50.964 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:17:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:50 smithi045 ceph-mon[95976]: osdmap e287: 8 total, 7 up, 8 in 2024-09-21T10:17:51.060 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:51.112 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:51.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:51.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:51.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:51.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:51.369 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:51.471 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:51.497 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:51.598 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:51.625 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:51.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:51.727 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:51.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.770 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:51.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:51.871 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:51.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:51 smithi032 ceph-mon[101299]: pgmap v846: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 12424/146832 objects degraded (8.461%); 14 MiB/s, 228 objects/s recovering 2024-09-21T10:17:51.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:51 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12424/146832 objects degraded (8.461%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:51.886 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:51 smithi032 ceph-mon[105748]: pgmap v846: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 12424/146832 objects degraded (8.461%); 14 MiB/s, 228 objects/s recovering 2024-09-21T10:17:51.886 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:51 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12424/146832 objects degraded (8.461%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:51.894 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:51.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:51.996 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:52.022 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:52.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:52.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:51 smithi045 ceph-mon[95976]: pgmap v846: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 12424/146832 objects degraded (8.461%); 14 MiB/s, 228 objects/s recovering 2024-09-21T10:17:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:51 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12424/146832 objects degraded (8.461%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:52.084 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:17:52.084 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:52.124 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:52.128 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:17:52.129 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:52.151 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:52.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:52.167 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:17:52.167 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:52.203 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:17:52.203 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:52.238 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:17:52.238 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:52.241 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:17:52.253 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:52.281 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:17:52.282 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:52.288 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:52.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:52.319 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:17:52.320 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:52.390 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:52.426 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:52.528 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:52.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:52.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:52.658 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:52.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:52.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:52.801 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:52.829 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:52.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:52.931 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:52.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:52 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1617861460' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:52.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:52 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1617861460' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:52.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.972 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:52.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:52 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1617861460' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:53.073 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:53.108 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:53.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:53.210 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:53.240 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:53.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:53.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:53.388 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:53.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:17:53.391 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:17:53.400 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:53.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:53.521 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:53.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:53.622 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:53.651 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:53.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:53.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:53 smithi032 ceph-mon[101299]: pgmap v847: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 12424/146139 objects degraded (8.501%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T10:17:53.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:53 smithi032 ceph-mon[105748]: pgmap v847: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 12424/146139 objects degraded (8.501%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T10:17:53.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:53.792 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:53.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:53.804 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:17:53.895 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:53.923 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:53.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:54.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:53 smithi045 ceph-mon[95976]: pgmap v847: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 12424/146139 objects degraded (8.501%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T10:17:54.104 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:54.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:54.205 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:54.253 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:54.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:54.355 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:54.382 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:54.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:54.483 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:54.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.523 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:54.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:54.623 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:54.653 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:54.754 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:54.783 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:54.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:54.885 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:54.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.923 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:54.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:55.024 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:55.057 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:55.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:55.158 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:55.193 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:55.292 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:17:55.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:55.297 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:17:55.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:55.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:55.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:17:55.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:55.466 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:55.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.615 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:55.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:55.716 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:55.738 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:55.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:55.840 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:55.868 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:55 smithi032 ceph-mon[105748]: pgmap v848: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 11922/141774 objects degraded (8.409%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T10:17:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:55 smithi032 ceph-mon[105748]: from='client.55123 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:17:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:55 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:17:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:55 smithi032 ceph-mon[101299]: pgmap v848: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 11922/141774 objects degraded (8.409%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T10:17:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:55 smithi032 ceph-mon[101299]: from='client.55123 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:17:55.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:55 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:17:55.970 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.000 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:56.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:55 smithi045 ceph-mon[95976]: pgmap v848: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 11922/141774 objects degraded (8.409%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T10:17:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:55 smithi045 ceph-mon[95976]: from='client.55123 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:17:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:55 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:17:56.101 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:56.142 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:56.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:56.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:56.283 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:56.385 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:56.419 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:56.520 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:56.562 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:56.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:56.665 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:56.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:56 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 11340/134115 objects degraded (8.455%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:56.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:56 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 11340/134115 objects degraded (8.455%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:56.713 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:56.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:56.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:56.853 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:56.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:56.996 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:56.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:57.011 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:56 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 11340/134115 objects degraded (8.455%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:57.098 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:57.131 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:57.232 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:57.287 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":848,"stamp":"2024-09-21T10:17:54.416895+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3099990013,"num_objects":47258,"num_object_clones":0,"num_object_copies":141774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11922,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47258,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":203221,"num_write_kb":7993474,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14335,"num_bytes_recovered":947064831,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128952,"ondisk_log_size":128952,"up":242,"acting":242,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13299328,"kb_used_data":9649452,"kb_used_omap":35,"kb_used_meta":3649244,"kb_avail":736661888,"statfs":{"total":767960285184,"available":754341773312,"internally_reserved":0,"allocated":9881038848,"data_stored":9870423744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":3736826574},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,2,3,2,0,0,0,3,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-362610688,"num_objects":-5533,"num_object_clones":0,"num_object_copies":-16599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1448,"num_objects_misplaced":-515,"num_objects_unfound":0,"num_objects_dirty":-5533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5533,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":496,"num_bytes_recovered":32505856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005016"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":449,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.567055+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:17:49.567055+0000","last_peered":"2024-09-21T10:17:49.567055+0000","last_clean":"2024-09-21T10:17:49.567055+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:17:49.567055+0000","last_undegraded":"2024-09-21T10:17:49.567055+0000","last_fullsized":"2024-09-21T10:17:49.567055+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":257,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.575357+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:17:48.575357+0000","last_peered":"2024-09-21T10:17:48.575357+0000","last_clean":"2024-09-21T10:17:48.575357+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:17:48.575357+0000","last_undegraded":"2024-09-21T10:17:48.575357+0000","last_fullsized":"2024-09-21T10:17:48.575357+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":179,"reported_epoch":286,"state":"active+undersized","last_fresh":"2024-09-21T10:17:48.575532+0000","last_change":"2024-09-21T10:17:35.175828+0000","last_active":"2024-09-21T10:17:48.575532+0000","last_peered":"2024-09-21T10:17:48.575532+0000","last_clean":"2024-09-21T10:03:19.759615+0000","last_became_active":"2024-09-21T10:17:35.175828+0000","last_became_peered":"2024-09-21T10:17:35.175828+0000","last_unstale":"2024-09-21T10:17:48.575532+0000","last_undegraded":"2024-09-21T10:17:48.575532+0000","last_fullsized":"2024-09-21T10:17:35.122679+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:34.208210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":189,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.575129+0000","last_change":"2024-09-21T10:13:09.583319+0000","last_active":"2024-09-21T10:17:48.575129+0000","last_peered":"2024-09-21T10:17:48.575129+0000","last_clean":"2024-09-21T10:17:48.575129+0000","last_became_active":"2024-09-21T10:13:09.582953+0000","last_became_peered":"2024-09-21T10:13:09.582953+0000","last_unstale":"2024-09-21T10:17:48.575129+0000","last_undegraded":"2024-09-21T10:17:48.575129+0000","last_fullsized":"2024-09-21T10:17:48.575129+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:13.630031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":276,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.567584+0000","last_change":"2024-09-21T10:09:35.958207+0000","last_active":"2024-09-21T10:17:49.567584+0000","last_peered":"2024-09-21T10:17:49.567584+0000","last_clean":"2024-09-21T10:17:49.567584+0000","last_became_active":"2024-09-21T10:09:35.957860+0000","last_became_peered":"2024-09-21T10:09:35.957860+0000","last_unstale":"2024-09-21T10:17:49.567584+0000","last_undegraded":"2024-09-21T10:17:49.567584+0000","last_fullsized":"2024-09-21T10:17:49.567584+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:37.767930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":240,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.575168+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:17:48.575168+0000","last_peered":"2024-09-21T10:17:48.575168+0000","last_clean":"2024-09-21T10:17:48.575168+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:17:48.575168+0000","last_undegraded":"2024-09-21T10:17:48.575168+0000","last_fullsized":"2024-09-21T10:17:48.575168+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":455,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.567601+0000","last_change":"2024-09-21T10:14:20.761360+0000","last_active":"2024-09-21T10:17:49.567601+0000","last_peered":"2024-09-21T10:17:49.567601+0000","last_clean":"2024-09-21T10:17:49.567601+0000","last_became_active":"2024-09-21T10:14:20.760941+0000","last_became_peered":"2024-09-21T10:14:20.760941+0000","last_unstale":"2024-09-21T10:17:49.567601+0000","last_undegraded":"2024-09-21T10:17:49.567601+0000","last_fullsized":"2024-09-21T10:17:49.567601+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:27.842256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":311,"reported_epoch":286,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:48.569307+0000","last_change":"2024-09-21T10:17:35.124435+0000","last_active":"2024-09-21T10:17:48.569307+0000","last_peered":"2024-09-21T10:17:48.569307+0000","last_clean":"2024-09-21T10:17:25.108942+0000","last_became_active":"2024-09-21T10:17:35.124435+0000","last_became_peered":"2024-09-21T10:17:35.124435+0000","last_unstale":"2024-09-21T10:17:48.569307+0000","last_undegraded":"2024-09-21T10:17:35.121018+0000","last_fullsized":"2024-09-21T10:17:35.120899+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:44.888283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"287'12565","reported_seq":12952,"reported_epoch":287,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:50.045514+0000","last_change":"2024-09-21T10:17:35.127762+0000","last_active":"2024-09-21T10:17:50.045514+0000","last_peered":"2024-09-21T10:17:50.045514+0000","last_clean":"2024-09-21T10:17:34.135019+0000","last_became_active":"2024-09-21T10:17:35.127762+0000","last_became_peered":"2024-09-21T10:17:35.127762+0000","last_unstale":"2024-09-21T10:17:50.045514+0000","last_undegraded":"2024-09-21T10:17:35.126607+0000","last_fullsized":"2024-09-21T10:17:35.126515+0000","mapping_epoch":281,"log_start":"264'5800","ondisk_log_start":"264'5800","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:44:22.448438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203358208,"num_objects":3103,"num_object_clones":0,"num_object_copies":9309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3103,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12565,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":9043968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":3103}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":296,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.577453+0000","last_change":"2024-09-21T10:13:09.586457+0000","last_active":"2024-09-21T10:17:48.577453+0000","last_peered":"2024-09-21T10:17:48.577453+0000","last_clean":"2024-09-21T10:17:48.577453+0000","last_became_active":"2024-09-21T10:13:09.574624+0000","last_became_peered":"2024-09-21T10:13:09.574624+0000","last_unstale":"2024-09-21T10:17:48.577453+0000","last_undegraded":"2024-09-21T10:17:48.577453+0000","last_fullsized":"2024-09-21T10:17:48.577453+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:39:00.778086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":117,"reported_epoch":286,"state":"active+undersized","last_fresh":"2024-09-21T10:17:48.575033+0000","last_change":"2024-09-21T10:17:35.175717+0000","last_active":"2024-09-21T10:17:48.575033+0000","last_peered":"2024-09-21T10:17:48.575033+0000","last_clean":"2024-09-21T09:50:22.381258+0000","last_became_active":"2024-09-21T10:17:35.175717+0000","last_became_peered":"2024-09-21T10:17:35.175717+0000","last_unstale":"2024-09-21T10:17:48.575033+0000","last_undegraded":"2024-09-21T10:17:48.575033+0000","last_fullsized":"2024-09-21T10:17:35.122234+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:22.621830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"287'12743","reported_seq":13163,"reported_epoch":287,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:53.342302+0000","last_change":"2024-09-21T10:17:35.130954+0000","last_active":"2024-09-21T10:17:53.342302+0000","last_peered":"2024-09-21T10:17:53.342302+0000","last_clean":"2024-09-21T10:17:34.113182+0000","last_became_active":"2024-09-21T10:17:35.130954+0000","last_became_peered":"2024-09-21T10:17:35.130954+0000","last_unstale":"2024-09-21T10:17:53.342302+0000","last_undegraded":"2024-09-21T10:17:35.121199+0000","last_fullsized":"2024-09-21T10:17:35.121083+0000","mapping_epoch":281,"log_start":"264'5600","ondisk_log_start":"264'5600","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7143,"log_dups_size":0,"ondisk_log_size":7143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:13:03.054152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193921024,"num_objects":2959,"num_object_clones":0,"num_object_copies":8877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2959,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12743,"num_write_kb":502464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":2959}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":323,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.575807+0000","last_change":"2024-09-21T10:13:09.596445+0000","last_active":"2024-09-21T10:17:48.575807+0000","last_peered":"2024-09-21T10:17:48.575807+0000","last_clean":"2024-09-21T10:17:48.575807+0000","last_became_active":"2024-09-21T10:13:09.582665+0000","last_became_peered":"2024-09-21T10:13:09.582665+0000","last_unstale":"2024-09-21T10:17:48.575807+0000","last_undegraded":"2024-09-21T10:17:48.575807+0000","last_fullsized":"2024-09-21T10:17:48.575807+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:23:47.510351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":288,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568280+0000","last_change":"2024-09-21T10:13:09.578819+0000","last_active":"2024-09-21T10:17:48.568280+0000","last_peered":"2024-09-21T10:17:48.568280+0000","last_clean":"2024-09-21T10:17:48.568280+0000","last_became_active":"2024-09-21T10:13:09.576707+0000","last_became_peered":"2024-09-21T10:13:09.576707+0000","last_unstale":"2024-09-21T10:17:48.568280+0000","last_undegraded":"2024-09-21T10:17:48.568280+0000","last_fullsized":"2024-09-21T10:17:48.568280+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:19.175846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"287'12627","reported_seq":12730,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:50.077247+0000","last_change":"2024-09-21T10:14:32.771740+0000","last_active":"2024-09-21T10:17:50.077247+0000","last_peered":"2024-09-21T10:17:50.077247+0000","last_clean":"2024-09-21T10:17:50.077247+0000","last_became_active":"2024-09-21T10:14:32.771298+0000","last_became_peered":"2024-09-21T10:14:32.771298+0000","last_unstale":"2024-09-21T10:17:50.077247+0000","last_undegraded":"2024-09-21T10:17:50.077247+0000","last_fullsized":"2024-09-21T10:17:50.077247+0000","mapping_epoch":248,"log_start":"264'5900","ondisk_log_start":"264'5900","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6727,"log_dups_size":0,"ondisk_log_size":6727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:27.287728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198901760,"num_objects":3035,"num_object_clones":0,"num_object_copies":9105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12627,"num_write_kb":501184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":274,"reported_epoch":286,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:48.575679+0000","last_change":"2024-09-21T10:17:35.175102+0000","last_active":"2024-09-21T10:17:48.575679+0000","last_peered":"2024-09-21T10:17:48.575679+0000","last_clean":"2024-09-21T10:17:25.110838+0000","last_became_active":"2024-09-21T10:17:35.175102+0000","last_became_peered":"2024-09-21T10:17:35.175102+0000","last_unstale":"2024-09-21T10:17:48.575679+0000","last_undegraded":"2024-09-21T10:17:35.122397+0000","last_fullsized":"2024-09-21T10:17:35.122269+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:57.830633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":273,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.569271+0000","last_change":"2024-09-21T10:13:09.578358+0000","last_active":"2024-09-21T10:17:48.569271+0000","last_peered":"2024-09-21T10:17:48.569271+0000","last_clean":"2024-09-21T10:17:48.569271+0000","last_became_active":"2024-09-21T10:13:09.577273+0000","last_became_peered":"2024-09-21T10:13:09.577273+0000","last_unstale":"2024-09-21T10:17:48.569271+0000","last_undegraded":"2024-09-21T10:17:48.569271+0000","last_fullsized":"2024-09-21T10:17:48.569271+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:17.971932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"287'12775","reported_seq":12871,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:53.989526+0000","last_change":"2024-09-21T10:14:28.438058+0000","last_active":"2024-09-21T10:17:53.989526+0000","last_peered":"2024-09-21T10:17:53.989526+0000","last_clean":"2024-09-21T10:17:53.989526+0000","last_became_active":"2024-09-21T10:14:28.437772+0000","last_became_peered":"2024-09-21T10:14:28.437772+0000","last_unstale":"2024-09-21T10:17:53.989526+0000","last_undegraded":"2024-09-21T10:17:53.989526+0000","last_fullsized":"2024-09-21T10:17:53.989526+0000","mapping_epoch":245,"log_start":"259'2700","ondisk_log_start":"259'2700","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":10075,"log_dups_size":0,"ondisk_log_size":10075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:14.437061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181075968,"num_objects":2763,"num_object_clones":0,"num_object_copies":8289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12775,"num_write_kb":497216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":83,"reported_epoch":287,"state":"active+undersized","last_fresh":"2024-09-21T10:17:49.567140+0000","last_change":"2024-09-21T10:17:35.125321+0000","last_active":"2024-09-21T10:17:49.567140+0000","last_peered":"2024-09-21T10:17:49.567140+0000","last_clean":"2024-09-21T10:03:19.757236+0000","last_became_active":"2024-09-21T10:17:35.125321+0000","last_became_peered":"2024-09-21T10:17:35.125321+0000","last_unstale":"2024-09-21T10:17:49.567140+0000","last_undegraded":"2024-09-21T10:17:49.567140+0000","last_fullsized":"2024-09-21T10:17:35.122818+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:18.261047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":46,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.569819+0000","last_change":"2024-09-21T10:16:35.252681+0000","last_active":"2024-09-21T10:17:49.569819+0000","last_peered":"2024-09-21T10:17:49.569819+0000","last_clean":"2024-09-21T10:17:49.569819+0000","last_became_active":"2024-09-21T10:16:35.252333+0000","last_became_peered":"2024-09-21T10:16:35.252333+0000","last_unstale":"2024-09-21T10:17:49.569819+0000","last_undegraded":"2024-09-21T10:17:49.569819+0000","last_fullsized":"2024-09-21T10:17:49.569819+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:43.211846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"287'12777","reported_seq":12873,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:53.343025+0000","last_change":"2024-09-21T10:14:28.441282+0000","last_active":"2024-09-21T10:17:53.343025+0000","last_peered":"2024-09-21T10:17:53.343025+0000","last_clean":"2024-09-21T10:17:53.343025+0000","last_became_active":"2024-09-21T10:14:28.440767+0000","last_became_peered":"2024-09-21T10:14:28.440767+0000","last_unstale":"2024-09-21T10:17:53.343025+0000","last_undegraded":"2024-09-21T10:17:53.343025+0000","last_fullsized":"2024-09-21T10:17:53.343025+0000","mapping_epoch":245,"log_start":"264'5600","ondisk_log_start":"264'5600","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7177,"log_dups_size":0,"ondisk_log_size":7177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:56:18.651559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181862400,"num_objects":2775,"num_object_clones":0,"num_object_copies":8325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12777,"num_write_kb":497664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":466,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.566972+0000","last_change":"2024-09-21T10:13:09.594302+0000","last_active":"2024-09-21T10:17:49.566972+0000","last_peered":"2024-09-21T10:17:49.566972+0000","last_clean":"2024-09-21T10:17:49.566972+0000","last_became_active":"2024-09-21T10:13:09.576948+0000","last_became_peered":"2024-09-21T10:13:09.576948+0000","last_unstale":"2024-09-21T10:17:49.566972+0000","last_undegraded":"2024-09-21T10:17:49.566972+0000","last_fullsized":"2024-09-21T10:17:49.566972+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:10.360060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":255,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568348+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:17:48.568348+0000","last_peered":"2024-09-21T10:17:48.568348+0000","last_clean":"2024-09-21T10:17:48.568348+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:17:48.568348+0000","last_undegraded":"2024-09-21T10:17:48.568348+0000","last_fullsized":"2024-09-21T10:17:48.568348+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"287'12881","reported_seq":13326,"reported_epoch":287,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:54.047480+0000","last_change":"2024-09-21T10:17:35.130144+0000","last_active":"2024-09-21T10:17:54.047480+0000","last_peered":"2024-09-21T10:17:54.047480+0000","last_clean":"2024-09-21T10:17:34.129564+0000","last_became_active":"2024-09-21T10:17:35.130144+0000","last_became_peered":"2024-09-21T10:17:35.130144+0000","last_unstale":"2024-09-21T10:17:54.047480+0000","last_undegraded":"2024-09-21T10:17:35.124645+0000","last_fullsized":"2024-09-21T10:17:35.124526+0000","mapping_epoch":281,"log_start":"259'2800","ondisk_log_start":"259'2800","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":10081,"log_dups_size":0,"ondisk_log_size":10081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:32.382462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182910976,"num_objects":2791,"num_object_clones":0,"num_object_copies":8373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2791,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12881,"num_write_kb":501504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":166,"num_bytes_recovered":10878976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2791}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":274,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.569421+0000","last_change":"2024-09-21T10:13:09.578151+0000","last_active":"2024-09-21T10:17:48.569421+0000","last_peered":"2024-09-21T10:17:48.569421+0000","last_clean":"2024-09-21T10:17:48.569421+0000","last_became_active":"2024-09-21T10:13:09.577233+0000","last_became_peered":"2024-09-21T10:13:09.577233+0000","last_unstale":"2024-09-21T10:17:48.569421+0000","last_undegraded":"2024-09-21T10:17:48.569421+0000","last_fullsized":"2024-09-21T10:17:48.569421+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:00.627590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":50,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568442+0000","last_change":"2024-09-21T10:14:32.771000+0000","last_active":"2024-09-21T10:17:48.568442+0000","last_peered":"2024-09-21T10:17:48.568442+0000","last_clean":"2024-09-21T10:17:48.568442+0000","last_became_active":"2024-09-21T10:14:32.770562+0000","last_became_peered":"2024-09-21T10:14:32.770562+0000","last_unstale":"2024-09-21T10:17:48.568442+0000","last_undegraded":"2024-09-21T10:17:48.568442+0000","last_fullsized":"2024-09-21T10:17:48.568442+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:47.956518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"287'12460","reported_seq":19597,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.744660+0000","last_change":"2024-09-21T10:17:49.596487+0000","last_active":"2024-09-21T10:17:49.744660+0000","last_peered":"2024-09-21T10:17:49.744660+0000","last_clean":"2024-09-21T10:17:49.744660+0000","last_became_active":"2024-09-21T10:17:49.571549+0000","last_became_peered":"2024-09-21T10:17:49.571549+0000","last_unstale":"2024-09-21T10:17:49.744660+0000","last_undegraded":"2024-09-21T10:17:49.744660+0000","last_fullsized":"2024-09-21T10:17:49.744660+0000","mapping_epoch":286,"log_start":"261'3600","ondisk_log_start":"261'3600","created":245,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":8860,"log_dups_size":0,"ondisk_log_size":8860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:07.398175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202309656,"num_objects":3088,"num_object_clones":0,"num_object_copies":9264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3088,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12460,"num_write_kb":497473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3528,"num_bytes_recovered":231145496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":256,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568619+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:17:48.568619+0000","last_peered":"2024-09-21T10:17:48.568619+0000","last_clean":"2024-09-21T10:17:48.568619+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:17:48.568619+0000","last_undegraded":"2024-09-21T10:17:48.568619+0000","last_fullsized":"2024-09-21T10:17:48.568619+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":273,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568545+0000","last_change":"2024-09-21T10:13:09.579444+0000","last_active":"2024-09-21T10:17:48.568545+0000","last_peered":"2024-09-21T10:17:48.568545+0000","last_clean":"2024-09-21T10:17:48.568545+0000","last_became_active":"2024-09-21T10:13:09.579162+0000","last_became_peered":"2024-09-21T10:13:09.579162+0000","last_unstale":"2024-09-21T10:17:48.568545+0000","last_undegraded":"2024-09-21T10:17:48.568545+0000","last_fullsized":"2024-09-21T10:17:48.568545+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.051244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"287'12419","reported_seq":12471,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.778879+0000","last_change":"2024-09-21T10:14:28.439503+0000","last_active":"2024-09-21T10:17:49.778879+0000","last_peered":"2024-09-21T10:17:49.778879+0000","last_clean":"2024-09-21T10:17:49.778879+0000","last_became_active":"2024-09-21T10:14:28.438413+0000","last_became_peered":"2024-09-21T10:14:28.438413+0000","last_unstale":"2024-09-21T10:17:49.778879+0000","last_undegraded":"2024-09-21T10:17:49.778879+0000","last_fullsized":"2024-09-21T10:17:49.778879+0000","mapping_epoch":245,"log_start":"261'3500","ondisk_log_start":"261'3500","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":8919,"log_dups_size":0,"ondisk_log_size":8919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:24.386364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197722112,"num_objects":3017,"num_object_clones":0,"num_object_copies":9051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12419,"num_write_kb":493952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":450,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.566683+0000","last_change":"2024-09-21T10:13:09.582151+0000","last_active":"2024-09-21T10:17:49.566683+0000","last_peered":"2024-09-21T10:17:49.566683+0000","last_clean":"2024-09-21T10:17:49.566683+0000","last_became_active":"2024-09-21T10:13:09.580224+0000","last_became_peered":"2024-09-21T10:13:09.580224+0000","last_unstale":"2024-09-21T10:17:49.566683+0000","last_undegraded":"2024-09-21T10:17:49.566683+0000","last_fullsized":"2024-09-21T10:17:49.566683+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:15.490461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":156,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.571099+0000","last_change":"2024-09-21T10:13:09.586643+0000","last_active":"2024-09-21T10:17:49.571099+0000","last_peered":"2024-09-21T10:17:49.571099+0000","last_clean":"2024-09-21T10:17:49.571099+0000","last_became_active":"2024-09-21T10:13:09.586283+0000","last_became_peered":"2024-09-21T10:13:09.586283+0000","last_unstale":"2024-09-21T10:17:49.571099+0000","last_undegraded":"2024-09-21T10:17:49.571099+0000","last_fullsized":"2024-09-21T10:17:49.571099+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:20.002373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"287'12776","reported_seq":21732,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:51.515548+0000","last_change":"2024-09-21T10:17:37.275700+0000","last_active":"2024-09-21T10:17:51.515548+0000","last_peered":"2024-09-21T10:17:51.515548+0000","last_clean":"2024-09-21T10:17:51.515548+0000","last_became_active":"2024-09-21T10:17:37.175690+0000","last_became_peered":"2024-09-21T10:17:37.175690+0000","last_unstale":"2024-09-21T10:17:51.515548+0000","last_undegraded":"2024-09-21T10:17:51.515548+0000","last_fullsized":"2024-09-21T10:17:51.515548+0000","mapping_epoch":283,"log_start":"263'4300","ondisk_log_start":"263'4300","created":245,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":8476,"log_dups_size":0,"ondisk_log_size":8476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:13:04.325145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193593344,"num_objects":2954,"num_object_clones":0,"num_object_copies":8862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12776,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4424,"num_bytes_recovered":289800192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":657,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.575269+0000","last_change":"2024-09-21T10:13:12.573143+0000","last_active":"2024-09-21T10:17:48.575269+0000","last_peered":"2024-09-21T10:17:48.575269+0000","last_clean":"2024-09-21T10:17:48.575269+0000","last_became_active":"2024-09-21T10:13:12.571751+0000","last_became_peered":"2024-09-21T10:13:12.571751+0000","last_unstale":"2024-09-21T10:17:48.575269+0000","last_undegraded":"2024-09-21T10:17:48.575269+0000","last_fullsized":"2024-09-21T10:17:48.575269+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T09:46:40.454871+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:28.804119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":394,"reported_epoch":287,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:49.598512+0000","last_change":"2024-09-21T10:17:35.178259+0000","last_active":"2024-09-21T10:17:49.598512+0000","last_peered":"2024-09-21T10:17:49.598512+0000","last_clean":"2024-09-21T09:51:58.517342+0000","last_became_active":"2024-09-21T10:17:35.178259+0000","last_became_peered":"2024-09-21T10:17:35.178259+0000","last_unstale":"2024-09-21T10:17:49.598512+0000","last_undegraded":"2024-09-21T10:17:35.125898+0000","last_fullsized":"2024-09-21T10:17:35.125796+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:32.882427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":401,"reported_epoch":287,"state":"active+undersized","last_fresh":"2024-09-21T10:17:49.566728+0000","last_change":"2024-09-21T10:17:35.126459+0000","last_active":"2024-09-21T10:17:49.566728+0000","last_peered":"2024-09-21T10:17:49.566728+0000","last_clean":"2024-09-21T10:17:25.100810+0000","last_became_active":"2024-09-21T10:17:35.126459+0000","last_became_peered":"2024-09-21T10:17:35.126459+0000","last_unstale":"2024-09-21T10:17:49.566728+0000","last_undegraded":"2024-09-21T10:17:49.566728+0000","last_fullsized":"2024-09-21T10:17:35.124041+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:02.350739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":241,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.569643+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:17:48.569643+0000","last_peered":"2024-09-21T10:17:48.569643+0000","last_clean":"2024-09-21T10:17:48.569643+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:17:48.569643+0000","last_undegraded":"2024-09-21T10:17:48.569643+0000","last_fullsized":"2024-09-21T10:17:48.569643+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"287'12509","reported_seq":12559,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.773912+0000","last_change":"2024-09-21T10:14:28.440396+0000","last_active":"2024-09-21T10:17:49.773912+0000","last_peered":"2024-09-21T10:17:49.773912+0000","last_clean":"2024-09-21T10:17:49.773912+0000","last_became_active":"2024-09-21T10:14:28.440137+0000","last_became_peered":"2024-09-21T10:14:28.440137+0000","last_unstale":"2024-09-21T10:17:49.773912+0000","last_undegraded":"2024-09-21T10:17:49.773912+0000","last_fullsized":"2024-09-21T10:17:49.773912+0000","mapping_epoch":245,"log_start":"263'4800","ondisk_log_start":"263'4800","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7709,"log_dups_size":0,"ondisk_log_size":7709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:33:10.456362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196280320,"num_objects":2995,"num_object_clones":0,"num_object_copies":8985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12509,"num_write_kb":496128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":924,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.575211+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:17:48.575211+0000","last_peered":"2024-09-21T10:17:48.575211+0000","last_clean":"2024-09-21T10:17:48.575211+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:17:48.575211+0000","last_undegraded":"2024-09-21T10:17:48.575211+0000","last_fullsized":"2024-09-21T10:17:48.575211+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":46,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.569765+0000","last_change":"2024-09-21T10:16:35.252630+0000","last_active":"2024-09-21T10:17:49.569765+0000","last_peered":"2024-09-21T10:17:49.569765+0000","last_clean":"2024-09-21T10:17:49.569765+0000","last_became_active":"2024-09-21T10:16:35.252279+0000","last_became_peered":"2024-09-21T10:16:35.252279+0000","last_unstale":"2024-09-21T10:17:49.569765+0000","last_undegraded":"2024-09-21T10:17:49.569765+0000","last_fullsized":"2024-09-21T10:17:49.569765+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:12.918989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":156,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.571120+0000","last_change":"2024-09-21T10:13:09.580568+0000","last_active":"2024-09-21T10:17:49.571120+0000","last_peered":"2024-09-21T10:17:49.571120+0000","last_clean":"2024-09-21T10:17:49.571120+0000","last_became_active":"2024-09-21T10:13:09.579053+0000","last_became_peered":"2024-09-21T10:13:09.579053+0000","last_unstale":"2024-09-21T10:17:49.571120+0000","last_undegraded":"2024-09-21T10:17:49.571120+0000","last_fullsized":"2024-09-21T10:17:49.571120+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:39.206612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"287'12481","reported_seq":12531,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.720897+0000","last_change":"2024-09-21T10:14:32.642821+0000","last_active":"2024-09-21T10:17:49.720897+0000","last_peered":"2024-09-21T10:17:49.720897+0000","last_clean":"2024-09-21T10:17:49.720897+0000","last_became_active":"2024-09-21T10:14:32.642404+0000","last_became_peered":"2024-09-21T10:14:32.642404+0000","last_unstale":"2024-09-21T10:17:49.720897+0000","last_undegraded":"2024-09-21T10:17:49.720897+0000","last_fullsized":"2024-09-21T10:17:49.720897+0000","mapping_epoch":248,"log_start":"263'4500","ondisk_log_start":"263'4500","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7981,"log_dups_size":0,"ondisk_log_size":7981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:02.051331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195362816,"num_objects":2981,"num_object_clones":0,"num_object_copies":8943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12481,"num_write_kb":494784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":998,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.598262+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:17:49.598262+0000","last_peered":"2024-09-21T10:17:49.598262+0000","last_clean":"2024-09-21T10:17:49.598262+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:17:49.598262+0000","last_undegraded":"2024-09-21T10:17:49.598262+0000","last_fullsized":"2024-09-21T10:17:49.598262+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":156,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.570911+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:17:49.570911+0000","last_peered":"2024-09-21T10:17:49.570911+0000","last_clean":"2024-09-21T10:17:49.570911+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:17:49.570911+0000","last_undegraded":"2024-09-21T10:17:49.570911+0000","last_fullsized":"2024-09-21T10:17:49.570911+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":160,"reported_epoch":287,"state":"active+undersized","last_fresh":"2024-09-21T10:17:49.571055+0000","last_change":"2024-09-21T10:17:35.177471+0000","last_active":"2024-09-21T10:17:49.571055+0000","last_peered":"2024-09-21T10:17:49.571055+0000","last_clean":"2024-09-21T10:17:30.051002+0000","last_became_active":"2024-09-21T10:17:35.177471+0000","last_became_peered":"2024-09-21T10:17:35.177471+0000","last_unstale":"2024-09-21T10:17:49.571055+0000","last_undegraded":"2024-09-21T10:17:49.571055+0000","last_fullsized":"2024-09-21T10:17:35.125981+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:08.374021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"287'12492","reported_seq":23754,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.764793+0000","last_change":"2024-09-21T10:17:12.636504+0000","last_active":"2024-09-21T10:17:49.764793+0000","last_peered":"2024-09-21T10:17:49.764793+0000","last_clean":"2024-09-21T10:17:49.764793+0000","last_became_active":"2024-09-21T10:17:12.615672+0000","last_became_peered":"2024-09-21T10:17:12.615672+0000","last_unstale":"2024-09-21T10:17:49.764793+0000","last_undegraded":"2024-09-21T10:17:49.764793+0000","last_fullsized":"2024-09-21T10:17:49.764793+0000","mapping_epoch":277,"log_start":"261'3600","ondisk_log_start":"261'3600","created":245,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":8892,"log_dups_size":0,"ondisk_log_size":8892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:54.239276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198180864,"num_objects":3024,"num_object_clones":0,"num_object_copies":9072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12492,"num_write_kb":496512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5594,"num_bytes_recovered":366608384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":986,"reported_epoch":287,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:49.570473+0000","last_change":"2024-09-21T10:17:35.127768+0000","last_active":"2024-09-21T10:17:49.570473+0000","last_peered":"2024-09-21T10:17:49.570473+0000","last_clean":"2024-09-21T10:12:59.503459+0000","last_became_active":"2024-09-21T10:17:35.127768+0000","last_became_peered":"2024-09-21T10:17:35.127768+0000","last_unstale":"2024-09-21T10:17:49.570473+0000","last_undegraded":"2024-09-21T10:17:35.126492+0000","last_fullsized":"2024-09-21T10:17:35.126379+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:37:30.134026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":189,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.598386+0000","last_change":"2024-09-21T10:16:35.253445+0000","last_active":"2024-09-21T10:17:49.598386+0000","last_peered":"2024-09-21T10:17:49.598386+0000","last_clean":"2024-09-21T10:17:49.598386+0000","last_became_active":"2024-09-21T10:16:35.253121+0000","last_became_peered":"2024-09-21T10:16:35.253121+0000","last_unstale":"2024-09-21T10:17:49.598386+0000","last_undegraded":"2024-09-21T10:17:49.598386+0000","last_fullsized":"2024-09-21T10:17:49.598386+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:36.816904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":156,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.570992+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:17:49.570992+0000","last_peered":"2024-09-21T10:17:49.570992+0000","last_clean":"2024-09-21T10:17:49.570992+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:17:49.570992+0000","last_undegraded":"2024-09-21T10:17:49.570992+0000","last_fullsized":"2024-09-21T10:17:49.570992+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"287'12404","reported_seq":12456,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.701084+0000","last_change":"2024-09-21T10:14:28.437169+0000","last_active":"2024-09-21T10:17:49.701084+0000","last_peered":"2024-09-21T10:17:49.701084+0000","last_clean":"2024-09-21T10:17:49.701084+0000","last_became_active":"2024-09-21T10:14:28.436873+0000","last_became_peered":"2024-09-21T10:14:28.436873+0000","last_unstale":"2024-09-21T10:17:49.701084+0000","last_undegraded":"2024-09-21T10:17:49.701084+0000","last_fullsized":"2024-09-21T10:17:49.701084+0000","mapping_epoch":245,"log_start":"263'4500","ondisk_log_start":"263'4500","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7904,"log_dups_size":0,"ondisk_log_size":7904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:16.516429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199491584,"num_objects":3044,"num_object_clones":0,"num_object_copies":9132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12404,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":99,"reported_epoch":287,"state":"active+undersized","last_fresh":"2024-09-21T10:17:49.567358+0000","last_change":"2024-09-21T10:17:35.132728+0000","last_active":"2024-09-21T10:17:49.567358+0000","last_peered":"2024-09-21T10:17:49.567358+0000","last_clean":"2024-09-21T10:03:19.766344+0000","last_became_active":"2024-09-21T10:17:35.132728+0000","last_became_peered":"2024-09-21T10:17:35.132728+0000","last_unstale":"2024-09-21T10:17:49.567358+0000","last_undegraded":"2024-09-21T10:17:49.567358+0000","last_fullsized":"2024-09-21T10:17:35.122807+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:44:50.875126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":765,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.571223+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:17:49.571223+0000","last_peered":"2024-09-21T10:17:49.571223+0000","last_clean":"2024-09-21T10:17:49.571223+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:17:49.571223+0000","last_undegraded":"2024-09-21T10:17:49.571223+0000","last_fullsized":"2024-09-21T10:17:49.571223+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":174,"reported_epoch":287,"state":"active+undersized","last_fresh":"2024-09-21T10:17:49.570772+0000","last_change":"2024-09-21T10:17:35.127413+0000","last_active":"2024-09-21T10:17:49.570772+0000","last_peered":"2024-09-21T10:17:49.570772+0000","last_clean":"2024-09-21T10:17:25.127289+0000","last_became_active":"2024-09-21T10:17:35.127413+0000","last_became_peered":"2024-09-21T10:17:35.127413+0000","last_unstale":"2024-09-21T10:17:49.570772+0000","last_undegraded":"2024-09-21T10:17:49.570772+0000","last_fullsized":"2024-09-21T10:17:35.126028+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:57.483438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"287'12662","reported_seq":13030,"reported_epoch":287,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:50.058143+0000","last_change":"2024-09-21T10:17:35.127386+0000","last_active":"2024-09-21T10:17:50.058143+0000","last_peered":"2024-09-21T10:17:50.058143+0000","last_clean":"2024-09-21T10:17:34.093837+0000","last_became_active":"2024-09-21T10:17:35.127386+0000","last_became_peered":"2024-09-21T10:17:35.127386+0000","last_unstale":"2024-09-21T10:17:50.058143+0000","last_undegraded":"2024-09-21T10:17:35.125205+0000","last_fullsized":"2024-09-21T10:17:35.125090+0000","mapping_epoch":281,"log_start":"264'5928","ondisk_log_start":"264'5928","created":245,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6734,"log_dups_size":0,"ondisk_log_size":6734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:42.679516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200409088,"num_objects":3058,"num_object_clones":0,"num_object_copies":9174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12662,"num_write_kb":503040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":3058}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":167,"reported_epoch":287,"state":"active+undersized","last_fresh":"2024-09-21T10:17:49.571185+0000","last_change":"2024-09-21T10:17:35.130553+0000","last_active":"2024-09-21T10:17:49.571185+0000","last_peered":"2024-09-21T10:17:49.571185+0000","last_clean":"2024-09-21T10:17:25.110962+0000","last_became_active":"2024-09-21T10:17:35.130553+0000","last_became_peered":"2024-09-21T10:17:35.130553+0000","last_unstale":"2024-09-21T10:17:49.571185+0000","last_undegraded":"2024-09-21T10:17:49.571185+0000","last_fullsized":"2024-09-21T10:17:35.127855+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:58.373232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":769,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.575717+0000","last_change":"2024-09-21T10:13:09.604050+0000","last_active":"2024-09-21T10:17:48.575717+0000","last_peered":"2024-09-21T10:17:48.575717+0000","last_clean":"2024-09-21T10:17:48.575717+0000","last_became_active":"2024-09-21T10:13:09.583593+0000","last_became_peered":"2024-09-21T10:13:09.583593+0000","last_unstale":"2024-09-21T10:17:48.575717+0000","last_undegraded":"2024-09-21T10:17:48.575717+0000","last_fullsized":"2024-09-21T10:17:48.575717+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:35.420105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":50,"reported_epoch":287,"state":"active+undersized","last_fresh":"2024-09-21T10:17:49.569782+0000","last_change":"2024-09-21T10:17:35.176265+0000","last_active":"2024-09-21T10:17:49.569782+0000","last_peered":"2024-09-21T10:17:49.569782+0000","last_clean":"2024-09-21T10:17:25.114921+0000","last_became_active":"2024-09-21T10:17:35.176265+0000","last_became_peered":"2024-09-21T10:17:35.176265+0000","last_unstale":"2024-09-21T10:17:49.569782+0000","last_undegraded":"2024-09-21T10:17:49.569782+0000","last_fullsized":"2024-09-21T10:17:35.128768+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:53.947774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"287'12870","reported_seq":12966,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:53.308847+0000","last_change":"2024-09-21T10:14:28.438913+0000","last_active":"2024-09-21T10:17:53.308847+0000","last_peered":"2024-09-21T10:17:53.308847+0000","last_clean":"2024-09-21T10:17:53.308847+0000","last_became_active":"2024-09-21T10:14:28.438598+0000","last_became_peered":"2024-09-21T10:14:28.438598+0000","last_unstale":"2024-09-21T10:17:53.308847+0000","last_undegraded":"2024-09-21T10:17:53.308847+0000","last_fullsized":"2024-09-21T10:17:53.308847+0000","mapping_epoch":245,"log_start":"264'5700","ondisk_log_start":"264'5700","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7170,"log_dups_size":0,"ondisk_log_size":7170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:29:24.765631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185729024,"num_objects":2834,"num_object_clones":0,"num_object_copies":8502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12870,"num_write_kb":502528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":455,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.567697+0000","last_change":"2024-09-21T10:16:35.254311+0000","last_active":"2024-09-21T10:17:49.567697+0000","last_peered":"2024-09-21T10:17:49.567697+0000","last_clean":"2024-09-21T10:17:49.567697+0000","last_became_active":"2024-09-21T10:16:35.250010+0000","last_became_peered":"2024-09-21T10:16:35.250010+0000","last_unstale":"2024-09-21T10:17:49.567697+0000","last_undegraded":"2024-09-21T10:17:49.567697+0000","last_fullsized":"2024-09-21T10:17:49.567697+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:54:13.306874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":760,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.571206+0000","last_change":"2024-09-21T10:13:12.569309+0000","last_active":"2024-09-21T10:17:49.571206+0000","last_peered":"2024-09-21T10:17:49.571206+0000","last_clean":"2024-09-21T10:17:49.571206+0000","last_became_active":"2024-09-21T10:13:12.568325+0000","last_became_peered":"2024-09-21T10:13:12.568325+0000","last_unstale":"2024-09-21T10:17:49.571206+0000","last_undegraded":"2024-09-21T10:17:49.571206+0000","last_fullsized":"2024-09-21T10:17:49.571206+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:18.314566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":452,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.566502+0000","last_change":"2024-09-21T10:13:09.579331+0000","last_active":"2024-09-21T10:17:49.566502+0000","last_peered":"2024-09-21T10:17:49.566502+0000","last_clean":"2024-09-21T10:17:49.566502+0000","last_became_active":"2024-09-21T10:13:09.578777+0000","last_became_peered":"2024-09-21T10:13:09.578777+0000","last_unstale":"2024-09-21T10:17:49.566502+0000","last_undegraded":"2024-09-21T10:17:49.566502+0000","last_fullsized":"2024-09-21T10:17:49.566502+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:50:34.425710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"287'12657","reported_seq":12753,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:53.308396+0000","last_change":"2024-09-21T10:14:28.439188+0000","last_active":"2024-09-21T10:17:53.308396+0000","last_peered":"2024-09-21T10:17:53.308396+0000","last_clean":"2024-09-21T10:17:53.308396+0000","last_became_active":"2024-09-21T10:14:28.438763+0000","last_became_peered":"2024-09-21T10:14:28.438763+0000","last_unstale":"2024-09-21T10:17:53.308396+0000","last_undegraded":"2024-09-21T10:17:53.308396+0000","last_fullsized":"2024-09-21T10:17:53.308396+0000","mapping_epoch":245,"log_start":"264'5500","ondisk_log_start":"264'5500","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7157,"log_dups_size":0,"ondisk_log_size":7157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:31.383586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184090624,"num_objects":2809,"num_object_clones":0,"num_object_copies":8427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12657,"num_write_kb":494912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":160,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.570971+0000","last_change":"2024-09-21T10:13:09.592519+0000","last_active":"2024-09-21T10:17:49.570971+0000","last_peered":"2024-09-21T10:17:49.570971+0000","last_clean":"2024-09-21T10:17:49.570971+0000","last_became_active":"2024-09-21T10:13:09.587615+0000","last_became_peered":"2024-09-21T10:13:09.587615+0000","last_unstale":"2024-09-21T10:17:49.570971+0000","last_undegraded":"2024-09-21T10:17:49.570971+0000","last_fullsized":"2024-09-21T10:17:49.570971+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:22:43.216612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":532,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.569828+0000","last_change":"2024-09-21T10:03:19.766519+0000","last_active":"2024-09-21T10:17:48.569828+0000","last_peered":"2024-09-21T10:17:48.569828+0000","last_clean":"2024-09-21T10:17:48.569828+0000","last_became_active":"2024-09-21T10:03:19.766017+0000","last_became_peered":"2024-09-21T10:03:19.766017+0000","last_unstale":"2024-09-21T10:17:48.569828+0000","last_undegraded":"2024-09-21T10:17:48.569828+0000","last_fullsized":"2024-09-21T10:17:48.569828+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:40.792313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":247,"reported_epoch":286,"state":"active+undersized","last_fresh":"2024-09-21T10:17:48.568398+0000","last_change":"2024-09-21T10:17:35.124983+0000","last_active":"2024-09-21T10:17:48.568398+0000","last_peered":"2024-09-21T10:17:48.568398+0000","last_clean":"2024-09-21T10:17:25.103643+0000","last_became_active":"2024-09-21T10:17:35.124983+0000","last_became_peered":"2024-09-21T10:17:35.124983+0000","last_unstale":"2024-09-21T10:17:48.568398+0000","last_undegraded":"2024-09-21T10:17:48.568398+0000","last_fullsized":"2024-09-21T10:17:35.123358+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:42:17.720690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":281,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568458+0000","last_change":"2024-09-21T10:15:21.038463+0000","last_active":"2024-09-21T10:17:48.568458+0000","last_peered":"2024-09-21T10:17:48.568458+0000","last_clean":"2024-09-21T10:17:48.568458+0000","last_became_active":"2024-09-21T10:15:21.038061+0000","last_became_peered":"2024-09-21T10:15:21.038061+0000","last_unstale":"2024-09-21T10:17:48.568458+0000","last_undegraded":"2024-09-21T10:17:48.568458+0000","last_fullsized":"2024-09-21T10:17:48.568458+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.241295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":263,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568623+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:17:48.568623+0000","last_peered":"2024-09-21T10:17:48.568623+0000","last_clean":"2024-09-21T10:17:48.568623+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:17:48.568623+0000","last_undegraded":"2024-09-21T10:17:48.568623+0000","last_fullsized":"2024-09-21T10:17:48.568623+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":97,"reported_epoch":286,"state":"active+undersized","last_fresh":"2024-09-21T10:17:48.569383+0000","last_change":"2024-09-21T10:17:35.128147+0000","last_active":"2024-09-21T10:17:48.569383+0000","last_peered":"2024-09-21T10:17:48.569383+0000","last_clean":"2024-09-21T10:15:13.965890+0000","last_became_active":"2024-09-21T10:17:35.128147+0000","last_became_peered":"2024-09-21T10:17:35.128147+0000","last_unstale":"2024-09-21T10:17:48.569383+0000","last_undegraded":"2024-09-21T10:17:48.569383+0000","last_fullsized":"2024-09-21T10:17:35.121096+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:48.657757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":258,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.575578+0000","last_change":"2024-09-21T10:03:53.841826+0000","last_active":"2024-09-21T10:17:48.575578+0000","last_peered":"2024-09-21T10:17:48.575578+0000","last_clean":"2024-09-21T10:17:48.575578+0000","last_became_active":"2024-09-21T10:03:53.841446+0000","last_became_peered":"2024-09-21T10:03:53.841446+0000","last_unstale":"2024-09-21T10:17:48.575578+0000","last_undegraded":"2024-09-21T10:17:48.575578+0000","last_fullsized":"2024-09-21T10:17:48.575578+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:05.622315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":174,"reported_epoch":286,"state":"active+undersized","last_fresh":"2024-09-21T10:17:48.575745+0000","last_change":"2024-09-21T10:17:35.175712+0000","last_active":"2024-09-21T10:17:48.575745+0000","last_peered":"2024-09-21T10:17:48.575745+0000","last_clean":"2024-09-21T10:12:59.521955+0000","last_became_active":"2024-09-21T10:17:35.175712+0000","last_became_peered":"2024-09-21T10:17:35.175712+0000","last_unstale":"2024-09-21T10:17:48.575745+0000","last_undegraded":"2024-09-21T10:17:48.575745+0000","last_fullsized":"2024-09-21T10:17:35.123603+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:48.523067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":383,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.567142+0000","last_change":"2024-09-21T10:03:53.845188+0000","last_active":"2024-09-21T10:17:49.567142+0000","last_peered":"2024-09-21T10:17:49.567142+0000","last_clean":"2024-09-21T10:17:49.567142+0000","last_became_active":"2024-09-21T10:03:53.843570+0000","last_became_peered":"2024-09-21T10:03:53.843570+0000","last_unstale":"2024-09-21T10:17:49.567142+0000","last_undegraded":"2024-09-21T10:17:49.567142+0000","last_fullsized":"2024-09-21T10:17:49.567142+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:29.390231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":50,"reported_epoch":287,"state":"active+undersized","last_fresh":"2024-09-21T10:17:49.570050+0000","last_change":"2024-09-21T10:17:35.131689+0000","last_active":"2024-09-21T10:17:49.570050+0000","last_peered":"2024-09-21T10:17:49.570050+0000","last_clean":"2024-09-21T10:17:25.115497+0000","last_became_active":"2024-09-21T10:17:35.131689+0000","last_became_peered":"2024-09-21T10:17:35.131689+0000","last_unstale":"2024-09-21T10:17:49.570050+0000","last_undegraded":"2024-09-21T10:17:49.570050+0000","last_fullsized":"2024-09-21T10:17:35.128889+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:53.272548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":462,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.566900+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:17:49.566900+0000","last_peered":"2024-09-21T10:17:49.566900+0000","last_clean":"2024-09-21T10:17:49.566900+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:17:49.566900+0000","last_undegraded":"2024-09-21T10:17:49.566900+0000","last_fullsized":"2024-09-21T10:17:49.566900+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":289,"reported_epoch":286,"state":"active+undersized","last_fresh":"2024-09-21T10:17:48.569784+0000","last_change":"2024-09-21T10:17:35.127626+0000","last_active":"2024-09-21T10:17:48.569784+0000","last_peered":"2024-09-21T10:17:48.569784+0000","last_clean":"2024-09-21T10:17:25.109952+0000","last_became_active":"2024-09-21T10:17:35.127626+0000","last_became_peered":"2024-09-21T10:17:35.127626+0000","last_unstale":"2024-09-21T10:17:48.569784+0000","last_undegraded":"2024-09-21T10:17:48.569784+0000","last_fullsized":"2024-09-21T10:17:35.121474+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.365796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":471,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.566584+0000","last_change":"2024-09-21T10:13:09.581932+0000","last_active":"2024-09-21T10:17:49.566584+0000","last_peered":"2024-09-21T10:17:49.566584+0000","last_clean":"2024-09-21T10:17:49.566584+0000","last_became_active":"2024-09-21T10:13:09.580798+0000","last_became_peered":"2024-09-21T10:13:09.580798+0000","last_unstale":"2024-09-21T10:17:49.566584+0000","last_undegraded":"2024-09-21T10:17:49.566584+0000","last_fullsized":"2024-09-21T10:17:49.566584+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:05.563826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":46,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.569970+0000","last_change":"2024-09-21T10:16:35.250798+0000","last_active":"2024-09-21T10:17:49.569970+0000","last_peered":"2024-09-21T10:17:49.569970+0000","last_clean":"2024-09-21T10:17:49.569970+0000","last_became_active":"2024-09-21T10:16:35.250461+0000","last_became_peered":"2024-09-21T10:16:35.250461+0000","last_unstale":"2024-09-21T10:17:49.569970+0000","last_undegraded":"2024-09-21T10:17:49.569970+0000","last_fullsized":"2024-09-21T10:17:49.569970+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:37.340319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":162,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.570432+0000","last_change":"2024-09-21T10:14:32.454735+0000","last_active":"2024-09-21T10:17:49.570432+0000","last_peered":"2024-09-21T10:17:49.570432+0000","last_clean":"2024-09-21T10:17:49.570432+0000","last_became_active":"2024-09-21T10:14:32.454475+0000","last_became_peered":"2024-09-21T10:14:32.454475+0000","last_unstale":"2024-09-21T10:17:49.570432+0000","last_undegraded":"2024-09-21T10:17:49.570432+0000","last_fullsized":"2024-09-21T10:17:49.570432+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:59.191649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":454,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.567203+0000","last_change":"2024-09-21T10:14:20.761163+0000","last_active":"2024-09-21T10:17:49.567203+0000","last_peered":"2024-09-21T10:17:49.567203+0000","last_clean":"2024-09-21T10:17:49.567203+0000","last_became_active":"2024-09-21T10:14:20.760814+0000","last_became_peered":"2024-09-21T10:14:20.760814+0000","last_unstale":"2024-09-21T10:17:49.567203+0000","last_undegraded":"2024-09-21T10:17:49.567203+0000","last_fullsized":"2024-09-21T10:17:49.567203+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:44.679743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":289,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.607168+0000","last_change":"2024-09-21T10:13:09.577994+0000","last_active":"2024-09-21T10:17:49.607168+0000","last_peered":"2024-09-21T10:17:49.607168+0000","last_clean":"2024-09-21T10:17:49.607168+0000","last_became_active":"2024-09-21T10:13:09.577311+0000","last_became_peered":"2024-09-21T10:13:09.577311+0000","last_unstale":"2024-09-21T10:17:49.607168+0000","last_undegraded":"2024-09-21T10:17:49.607168+0000","last_fullsized":"2024-09-21T10:17:49.607168+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:27.532202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":467,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.598541+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:17:49.598541+0000","last_peered":"2024-09-21T10:17:49.598541+0000","last_clean":"2024-09-21T10:17:49.598541+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:17:49.598541+0000","last_undegraded":"2024-09-21T10:17:49.598541+0000","last_fullsized":"2024-09-21T10:17:49.598541+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":168,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568364+0000","last_change":"2024-09-21T10:13:09.578314+0000","last_active":"2024-09-21T10:17:48.568364+0000","last_peered":"2024-09-21T10:17:48.568364+0000","last_clean":"2024-09-21T10:17:48.568364+0000","last_became_active":"2024-09-21T10:13:09.577570+0000","last_became_peered":"2024-09-21T10:13:09.577570+0000","last_unstale":"2024-09-21T10:17:48.568364+0000","last_undegraded":"2024-09-21T10:17:48.568364+0000","last_fullsized":"2024-09-21T10:17:48.568364+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:21.299713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":134,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568266+0000","last_change":"2024-09-21T10:13:09.577841+0000","last_active":"2024-09-21T10:17:48.568266+0000","last_peered":"2024-09-21T10:17:48.568266+0000","last_clean":"2024-09-21T10:17:48.568266+0000","last_became_active":"2024-09-21T10:13:09.576793+0000","last_became_peered":"2024-09-21T10:13:09.576793+0000","last_unstale":"2024-09-21T10:17:48.568266+0000","last_undegraded":"2024-09-21T10:17:48.568266+0000","last_fullsized":"2024-09-21T10:17:48.568266+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:12.718520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":26,"reported_epoch":286,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:48.568477+0000","last_change":"2024-09-21T10:17:35.128563+0000","last_active":"2024-09-21T10:17:48.568477+0000","last_peered":"2024-09-21T10:17:48.568477+0000","last_clean":"2024-09-21T09:50:27.027024+0000","last_became_active":"2024-09-21T10:17:35.128563+0000","last_became_peered":"2024-09-21T10:17:35.128563+0000","last_unstale":"2024-09-21T10:17:48.568477+0000","last_undegraded":"2024-09-21T10:17:35.123737+0000","last_fullsized":"2024-09-21T10:17:35.123629+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:24:08.945613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":59,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568104+0000","last_change":"2024-09-21T10:16:35.252818+0000","last_active":"2024-09-21T10:17:48.568104+0000","last_peered":"2024-09-21T10:17:48.568104+0000","last_clean":"2024-09-21T10:17:48.568104+0000","last_became_active":"2024-09-21T10:16:35.252497+0000","last_became_peered":"2024-09-21T10:16:35.252497+0000","last_unstale":"2024-09-21T10:17:48.568104+0000","last_undegraded":"2024-09-21T10:17:48.568104+0000","last_fullsized":"2024-09-21T10:17:48.568104+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:30.018576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":26,"reported_epoch":286,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:48.568636+0000","last_change":"2024-09-21T10:17:35.124641+0000","last_active":"2024-09-21T10:17:48.568636+0000","last_peered":"2024-09-21T10:17:48.568636+0000","last_clean":"2024-09-21T09:50:27.027148+0000","last_became_active":"2024-09-21T10:17:35.124641+0000","last_became_peered":"2024-09-21T10:17:35.124641+0000","last_unstale":"2024-09-21T10:17:48.568636+0000","last_undegraded":"2024-09-21T10:17:35.122956+0000","last_fullsized":"2024-09-21T10:17:35.122826+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:05.334896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":481,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.566763+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:17:49.566763+0000","last_peered":"2024-09-21T10:17:49.566763+0000","last_clean":"2024-09-21T10:17:49.566763+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:17:49.566763+0000","last_undegraded":"2024-09-21T10:17:49.566763+0000","last_fullsized":"2024-09-21T10:17:49.566763+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":443,"reported_epoch":287,"state":"active+clean","last_fresh":"2024-09-21T10:17:49.567670+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:17:49.567670+0000","last_peered":"2024-09-21T10:17:49.567670+0000","last_clean":"2024-09-21T10:17:49.567670+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:17:49.567670+0000","last_undegraded":"2024-09-21T10:17:49.567670+0000","last_fullsized":"2024-09-21T10:17:49.567670+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":272,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-09-21T10:17:48.568049+0000","last_change":"2024-09-21T10:13:09.577388+0000","last_active":"2024-09-21T10:17:48.568049+0000","last_peered":"2024-09-21T10:17:48.568049+0000","last_clean":"2024-09-21T10:17:48.568049+0000","last_became_active":"2024-09-21T10:13:09.577027+0000","last_became_peered":"2024-09-21T10:13:09.577027+0000","last_unstale":"2024-09-21T10:17:48.568049+0000","last_undegraded":"2024-09-21T10:17:48.568049+0000","last_fullsized":"2024-09-21T10:17:48.568049+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:49.522609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":184,"reported_epoch":287,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:17:49.598803+0000","last_change":"2024-09-21T10:17:35.129614+0000","last_active":"2024-09-21T10:17:49.598803+0000","last_peered":"2024-09-21T10:17:49.598803+0000","last_clean":"2024-09-21T10:14:43.015500+0000","last_became_active":"2024-09-21T10:17:35.129614+0000","last_became_peered":"2024-09-21T10:17:35.129614+0000","last_unstale":"2024-09-21T10:17:49.598803+0000","last_undegraded":"2024-09-21T10:17:35.124679+0000","last_fullsized":"2024-09-21T10:17:35.124462+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:00:36.906750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3095199768,"num_objects":47230,"num_object_clones":0,"num_object_copies":141690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11911,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47230,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202098,"num_write_kb":7978433,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14144,"num_bytes_recovered":926744600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9795350528,"data_stored":9795338312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127770,"ondisk_log_size":127770,"up":44,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":88,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":788,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":85,"acting":85,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":152,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":22,"acting":22,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234338,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1677232,"kb_used_data":1168676,"kb_used_omap":1,"kb_used_meta":508542,"kb_avail":92067920,"statfs":{"total":95995035648,"available":94277550080,"internally_reserved":0,"allocated":1196724224,"data_stored":1195386634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":520747467},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495976,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1613552,"kb_used_data":1177192,"kb_used_omap":1,"kb_used_meta":436350,"kb_avail":92131600,"statfs":{"total":95995035648,"available":94342758400,"internally_reserved":0,"allocated":1205444608,"data_stored":1204103732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":446822855},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,3,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757609,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1670740,"kb_used_data":1274572,"kb_used_omap":1,"kb_used_meta":396158,"kb_avail":92074412,"statfs":{"total":95995035648,"available":94284197888,"internally_reserved":0,"allocated":1305161728,"data_stored":1303858859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":405666246},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019249,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1578680,"kb_used_data":1164460,"kb_used_omap":1,"kb_used_meta":414206,"kb_avail":92166472,"statfs":{"total":95995035648,"available":94378467328,"internally_reserved":0,"allocated":1192407040,"data_stored":1191069301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":424147398},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435434,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2400800,"kb_used_data":1837016,"kb_used_omap":5,"kb_used_meta":563386,"kb_avail":91344352,"statfs":{"total":95995035648,"available":93536616448,"internally_reserved":0,"allocated":1881104384,"data_stored":1879782325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":576907304},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598961,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1646496,"kb_used_data":1083032,"kb_used_omap":7,"kb_used_meta":563384,"kb_avail":92098656,"statfs":{"total":95995035648,"available":94309023744,"internally_reserved":0,"allocated":1109024768,"data_stored":1107702927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":576905677},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762487,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1779500,"kb_used_data":1364576,"kb_used_omap":7,"kb_used_meta":414840,"kb_avail":91965652,"statfs":{"total":95995035648,"available":94172827648,"internally_reserved":0,"allocated":1397325824,"data_stored":1395995819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":424796413},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926014,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":932328,"kb_used_data":579928,"kb_used_omap":7,"kb_used_meta":352376,"kb_avail":92812824,"statfs":{"total":95995035648,"available":95040331776,"internally_reserved":0,"allocated":593846272,"data_stored":592524147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":360833214},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":580849664,"data_stored":580845592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1388711936,"data_stored":1388707864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1100218368,"data_stored":1100218368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1862598656,"data_stored":1862598656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1186136064,"data_stored":1186136064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1295253504,"data_stored":1295253504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1196625920,"data_stored":1196621848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1184956416,"data_stored":1184956416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:17:57.291 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:17:57.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pg_num 11 2024-09-21T10:17:57.305 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:57.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:57.407 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:57.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.449 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:57.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:57.550 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:57.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:57.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:57.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:57.729 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:57.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:57.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.875 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:57.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:57.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:57 smithi032 ceph-mon[101299]: pgmap v849: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 11340/134115 objects degraded (8.455%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T10:17:57.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:57 smithi032 ceph-mon[105748]: pgmap v849: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 11340/134115 objects degraded (8.455%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T10:17:57.976 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:58.017 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:57 smithi045 ceph-mon[95976]: pgmap v849: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 11340/134115 objects degraded (8.455%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T10:17:58.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.080 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:58.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:58.181 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:58.240 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:58.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:58.323 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:58.341 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:58.394 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:58.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:58.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:58.525 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:58.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:58.627 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:58.669 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:58.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:58.810 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:58.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:58.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.958 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:58.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:58.976 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:58.976 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:59.059 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:59.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.118 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:59.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:59.219 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:59.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:59.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:59.362 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:59.399 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:59.501 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:59.531 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:59.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:59.633 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:59.672 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:17:59.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:59.773 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:59.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.816 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:59.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:59.917 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:59.941 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:17:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:59.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:17:59 smithi032 ceph-mon[105748]: pgmap v850: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 11044/129903 objects degraded (8.502%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:17:59.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:17:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:17:59] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.51.0" 2024-09-21T10:17:59.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:17:59 smithi032 ceph-mon[101299]: pgmap v850: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 11044/129903 objects degraded (8.502%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:18:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:17:59 smithi045 ceph-mon[95976]: pgmap v850: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 11044/129903 objects degraded (8.502%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:18:00.043 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:00.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.082 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:00.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:00.183 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:00.206 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:00.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:00.308 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:00.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:00.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:00.455 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:00.492 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:00.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:00.594 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:00.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:00.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:00.743 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:00.796 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:00.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:00.898 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:00.938 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:00.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:01.040 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:01.086 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:01.187 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:01.214 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:01.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:01.316 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:01.343 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:01.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:01.445 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:01.483 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:01.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:01.585 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:01.641 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:18:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.658 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:01.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:01.759 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:01.802 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:01.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:01.882 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:01 smithi032 ceph-mon[101299]: pgmap v851: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 10234/120915 objects degraded (8.464%); 2.9 MiB/s, 45 objects/s recovering 2024-09-21T10:18:01.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:01 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:01.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:01 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:01.883 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:01 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10234/120915 objects degraded (8.464%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:01.883 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:01 smithi032 ceph-mon[105748]: pgmap v851: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 10234/120915 objects degraded (8.464%); 2.9 MiB/s, 45 objects/s recovering 2024-09-21T10:18:01.883 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:01 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:01.883 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:01 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:01.883 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:01 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10234/120915 objects degraded (8.464%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:01.903 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:01.927 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:01.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:02.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:01 smithi045 ceph-mon[95976]: pgmap v851: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 10234/120915 objects degraded (8.464%); 2.9 MiB/s, 45 objects/s recovering 2024-09-21T10:18:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:01 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:01 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:01 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10234/120915 objects degraded (8.464%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:02.056 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:02.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:02.158 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:02.182 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:02.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:02.283 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:02.305 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:02.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:02.407 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:02.431 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:02.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:02.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:02.571 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:02.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:02.673 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:02.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:02.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:02.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.857 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:02.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:02.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:02.981 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:02.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:03.083 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:03.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.121 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:03.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:03.221 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:03.254 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:03.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:03.355 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:03.384 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:03.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:03.485 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:03.530 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:03.632 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:03.669 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:03.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:03.771 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:03.797 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:03 smithi032 ceph-mon[101299]: pgmap v852: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 10234/120075 objects degraded (8.523%) 2024-09-21T10:18:03.797 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:03.797 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:03 smithi032 ceph-mon[105748]: pgmap v852: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 10234/120075 objects degraded (8.523%) 2024-09-21T10:18:03.797 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:03.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:03.912 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:03.939 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:03.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:03 smithi045 ceph-mon[95976]: pgmap v852: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 10234/120075 objects degraded (8.523%) 2024-09-21T10:18:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:04.040 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:04.085 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:04.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:04.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:04.327 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:04.359 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:04.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:04.461 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:04.489 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:04.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:04.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:04.636 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:04.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:04.738 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:04.772 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:04.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:04.874 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:04.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.909 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:04.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:05.010 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:05.038 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:05.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:05.139 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:05.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:05.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:05.263 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:05.286 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:05.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:05.427 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:05.528 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:05.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:05.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:05.655 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:18:05.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:05.680 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:05.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:05.781 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:05.808 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:05.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:05.910 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:05.945 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:05.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:05 smithi032 ceph-mon[105748]: pgmap v853: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 9726/115653 objects degraded (8.410%) 2024-09-21T10:18:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/253022857' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:05 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:05 smithi032 ceph-mon[101299]: pgmap v853: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 9726/115653 objects degraded (8.410%) 2024-09-21T10:18:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/253022857' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:05 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:05 smithi045 ceph-mon[95976]: pgmap v853: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 9726/115653 objects degraded (8.410%) 2024-09-21T10:18:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/253022857' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:05 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:06.047 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:06.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:06.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:06.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:06.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:06.320 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:06.356 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:06.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:06.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:06.599 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:06.628 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:06.730 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:06.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:06.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:06.871 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:06.902 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:06.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:06.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:06 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:06.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:06 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:18:06.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:06 smithi032 ceph-mon[105748]: osdmap e288: 8 total, 7 up, 8 in 2024-09-21T10:18:06.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:06 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 9726/115653 objects degraded (8.410%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:06 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:06 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:18:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:06 smithi032 ceph-mon[101299]: osdmap e288: 8 total, 7 up, 8 in 2024-09-21T10:18:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:06 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 9726/115653 objects degraded (8.410%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:07.004 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:07.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:07.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:06 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:06 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:18:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:06 smithi045 ceph-mon[95976]: osdmap e288: 8 total, 7 up, 8 in 2024-09-21T10:18:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:06 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 9726/115653 objects degraded (8.410%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:07.133 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:07.164 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:07.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:07.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:07.299 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:07.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:07.401 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:07.417 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:07.450 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:07.481 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:07.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:07.605 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:07.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:07.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:07.783 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:07.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:07.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:07 smithi032 ceph-mon[105748]: pgmap v855: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 9239/109062 objects degraded (8.471%) 2024-09-21T10:18:07.843 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:07 smithi032 ceph-mon[101299]: pgmap v855: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 9239/109062 objects degraded (8.471%) 2024-09-21T10:18:07.884 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:07.911 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:07.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:08.012 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:07 smithi045 ceph-mon[95976]: pgmap v855: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 9239/109062 objects degraded (8.471%) 2024-09-21T10:18:08.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:08.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:08.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.182 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:08.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:08.282 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:08.321 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:08.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:08.423 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:08.453 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:08.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.583 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:08.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:08.684 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:08.684 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 1 pg_num to 11 2024-09-21T10:18:08.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:08.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:08.812 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:08.844 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:08.946 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:08.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:08 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4081967591' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T10:18:08.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:08 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4081967591' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T10:18:08.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:08.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:09.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:08 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4081967591' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T10:18:09.075 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:09.112 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:09.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:09.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:18:09.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:09.244 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:09.345 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:09.373 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:09.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:09.475 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:09.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:09.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:09.612 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:09.641 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:09.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:09.742 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:09.776 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:09.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:09.911 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:09 smithi032 ceph-mon[105748]: pgmap v856: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9239/108324 objects degraded (8.529%) 2024-09-21T10:18:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4081967591' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-21T10:18:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:09 smithi032 ceph-mon[105748]: osdmap e289: 8 total, 7 up, 8 in 2024-09-21T10:18:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:18:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:18:09] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-09-21T10:18:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:09 smithi032 ceph-mon[101299]: pgmap v856: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9239/108324 objects degraded (8.529%) 2024-09-21T10:18:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4081967591' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-21T10:18:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:09 smithi032 ceph-mon[101299]: osdmap e289: 8 total, 7 up, 8 in 2024-09-21T10:18:10.013 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:09 smithi045 ceph-mon[95976]: pgmap v856: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9239/108324 objects degraded (8.529%) 2024-09-21T10:18:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4081967591' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-21T10:18:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:09 smithi045 ceph-mon[95976]: osdmap e289: 8 total, 7 up, 8 in 2024-09-21T10:18:10.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.061 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:10.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:10.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:10.190 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:10.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:10.292 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:10.319 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:10.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:10.420 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:10.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:10.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:10.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:10.595 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:10.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:10.697 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:10.726 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:10.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:10.828 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:10.852 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:10.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:10.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-21T10:18:10.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-21T10:18:10.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:10.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:11.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-21T10:18:11.081 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:11.123 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:11.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:11.224 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:11.253 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:11.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:11.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:11.391 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:11.492 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:11.524 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:11.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:11.626 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:11.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.691 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:11.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:11.792 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:11.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.822 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:11.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:11.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:11 smithi032 ceph-mon[101299]: pgmap v858: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 8081/95907 objects degraded (8.426%) 2024-09-21T10:18:11.832 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:11 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-21T10:18:11.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:11 smithi032 ceph-mon[101299]: osdmap e290: 8 total, 7 up, 8 in 2024-09-21T10:18:11.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:11 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 8081/95907 objects degraded (8.426%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:11.833 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:11 smithi032 ceph-mon[105748]: pgmap v858: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 8081/95907 objects degraded (8.426%) 2024-09-21T10:18:11.833 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:11 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-21T10:18:11.833 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:11 smithi032 ceph-mon[105748]: osdmap e290: 8 total, 7 up, 8 in 2024-09-21T10:18:11.833 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:11 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 8081/95907 objects degraded (8.426%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:11.922 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:11.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.952 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:11.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:11 smithi045 ceph-mon[95976]: pgmap v858: 89 pgs: 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 8081/95907 objects degraded (8.426%) 2024-09-21T10:18:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:11 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-21T10:18:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:11 smithi045 ceph-mon[95976]: osdmap e290: 8 total, 7 up, 8 in 2024-09-21T10:18:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:11 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 8081/95907 objects degraded (8.426%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:12.054 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:12.082 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:12.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:12.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:12.192 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:18:12.192 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:18:12.236 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:18:12.236 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:12.243 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:12.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:12.269 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:18:12.269 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:12.320 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:18:12.320 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:12.346 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:12.362 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:18:12.363 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:12.364 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:18:12.372 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:12.407 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:18:12.407 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:12.450 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:18:12.451 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:12.474 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.509 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:12.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:12.609 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:12.650 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:220604 7.9 7.a17e3449 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:12.706932+0000", 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0016299509999999999, 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0016637589999999999, 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:12.716 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 220604 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706932+0000", 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706932+0000", 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706933+0000", 2024-09-21T10:18:12.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.3999999999999997e-07 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706935+0000", 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.041e-06 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706940+0000", 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7379999999999999e-06 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.718 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706945+0000", 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6199999999999998e-06 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:220600 7.2 e291/245 7:467d2ce7:::benchmark_data_smithi032_200324_object95934:head v 291'13690, mlcod=291'13688)", 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:12.706957+0000", 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0016056410000000001, 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0016953370000000001, 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:12.719 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706957+0000", 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706957+0000", 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706957+0000", 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6600000000000002e-07 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706961+0000", 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9550000000000002e-06 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706967+0000", 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3069999999999999e-06 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706972+0000", 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1590000000000001e-06 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.707126+0000", 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015319899999999999 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.707146+0000", 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0760000000000001e-05 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.45206.0:220590 7.9 e291/248)", 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:12.708373+0000", 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.000189712, 2024-09-21T10:18:12.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000325414, 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708373+0000", 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708373+0000", 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708374+0000", 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6599999999999994e-07 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708379+0000", 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0769999999999998e-06 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708389+0000", 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0627e-05 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.724 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708396+0000", 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7340000000000002e-06 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:220603 7.5 e291/245 7:a6f33a86:::benchmark_data_smithi032_200324_object95937:head v 291'13707, mlcod=291'13705)", 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:12.708408+0000", 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.00015398300000000001, 2024-09-21T10:18:12.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00032833999999999999, 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708408+0000", 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708408+0000", 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708409+0000", 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0600000000000001e-07 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708413+0000", 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9160000000000003e-06 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708419+0000", 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0900000000000001e-06 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:12.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708423+0000", 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.233e-06 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708514+0000", 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1088000000000006e-05 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708532+0000", 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7640999999999999e-05 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.728 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:220590 7.9 7:9902f471:::benchmark_data_smithi032_200324_object95924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:12.694130+0000", 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.014432367999999999, 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.014658841000000001, 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 220590 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:12.729 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.694130+0000", 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.694130+0000", 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.694130+0000", 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1600000000000002e-07 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.694133+0000", 2024-09-21T10:18:12.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5469999999999999e-06 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.694139+0000", 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3759999999999996e-06 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.694145+0000", 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3499999999999996e-06 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.703811+0000", 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0096664690000000004 2024-09-21T10:18:12.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.703928+0000", 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000116521 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.704037+0000", 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000109622 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.707321+0000", 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0032838350000000001 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.732 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.707368+0000", 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7023e-05 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:220596 7.d e291/286 7:b7069cf9:::benchmark_data_smithi032_200324_object95930:head v 291'13755, mlcod=291'13752)", 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:12.699760+0000", 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0088021869999999995, 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0090970909999999999, 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:12.733 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.699760+0000", 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.699760+0000", 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.699761+0000", 2024-09-21T10:18:12.734 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1659999999999999e-06 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.699768+0000", 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0539999999999997e-06 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.699784+0000", 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5956000000000001e-05 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.699792+0000", 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6450000000000005e-06 2024-09-21T10:18:12.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.704832+0000", 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0050401509999999997 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.704857+0000", 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5190000000000001e-05 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.736 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:220598 7.d e291/286 7:bae72393:::benchmark_data_smithi032_200324_object95932:head v 291'13756, mlcod=291'13752)", 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:12.703005+0000", 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0055574609999999997, 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0058943609999999999, 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.703005+0000", 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.703005+0000", 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.703006+0000", 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5899999999999995e-07 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.703011+0000", 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6810000000000003e-06 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.738 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.703022+0000", 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1039999999999999e-05 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.703029+0000", 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8449999999999997e-06 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708341+0000", 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0053113850000000001 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.739 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.708359+0000", 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8603000000000001e-05 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:220599 7.9 7:98557f80:::benchmark_data_smithi032_200324_object95933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:12.706902+0000", 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.001659993, 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0020452859999999999, 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:18:12.740 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 220599 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706902+0000", 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706902+0000", 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706904+0000", 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.393e-06 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706907+0000", 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.405e-06 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706916+0000", 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7250000000000003e-06 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:12.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.706921+0000", 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5740000000000003e-06 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.707398+0000", 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00047686799999999998 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.707513+0000", 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114318 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:12.707627+0000", 2024-09-21T10:18:12.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114762 2024-09-21T10:18:12.744 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.744 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:12.744 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.744 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:12.744 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:18:12.744 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 8 2024-09-21T10:18:12.744 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:18:12.752 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:12.789 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:12.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:12.933 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:12.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:12 smithi032 ceph-mon[105748]: osdmap e291: 8 total, 7 up, 8 in 2024-09-21T10:18:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:12 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/985162611' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:12.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:12 smithi032 ceph-mon[101299]: osdmap e291: 8 total, 7 up, 8 in 2024-09-21T10:18:12.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:12 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/985162611' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:13.034 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:13.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:12 smithi045 ceph-mon[95976]: osdmap e291: 8 total, 7 up, 8 in 2024-09-21T10:18:13.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:12 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/985162611' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:13.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.060 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:13.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:13.161 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:13.200 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:13.301 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:13.331 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:13.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:13.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:13.471 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:13.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:13.572 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:13.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:13.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:18:13.619 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:18:13.701 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:13.739 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:13.841 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:13.869 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:13.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:13.908 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:13 smithi032 ceph-mon[101299]: pgmap v861: 99 pgs: 10 unknown, 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 8081/95049 objects degraded (8.502%) 2024-09-21T10:18:13.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:13 smithi032 ceph-mon[105748]: pgmap v861: 99 pgs: 10 unknown, 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 8081/95049 objects degraded (8.502%) 2024-09-21T10:18:13.971 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:14.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.016 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:14.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:13 smithi045 ceph-mon[95976]: pgmap v861: 99 pgs: 10 unknown, 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 8081/95049 objects degraded (8.502%) 2024-09-21T10:18:14.116 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:14.151 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:14.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:14.253 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:14.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.289 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:14.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:14.385 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:18:14.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:14.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:14.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:14.517 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:14.542 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:14.644 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.691 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:14.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:14.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:14.837 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:14.939 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:14.989 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:14.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:15.090 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:15.120 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:15.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:15.244 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:15.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:15.346 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:15.376 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:15.478 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:15.520 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:15.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:15.620 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:18:15.622 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:15.653 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:15.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:15.755 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:15.802 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:15.904 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:15.936 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:15.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:15.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:15 smithi032 ceph-mon[105748]: pgmap v862: 99 pgs: 10 unknown, 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 7476/89724 objects degraded (8.332%) 2024-09-21T10:18:15.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:15 smithi032 ceph-mon[105748]: 1.0 scrub starts 2024-09-21T10:18:15.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:15 smithi032 ceph-mon[105748]: 1.0 scrub ok 2024-09-21T10:18:15.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:15 smithi032 ceph-mon[101299]: pgmap v862: 99 pgs: 10 unknown, 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 7476/89724 objects degraded (8.332%) 2024-09-21T10:18:15.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:15 smithi032 ceph-mon[101299]: 1.0 scrub starts 2024-09-21T10:18:15.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:15 smithi032 ceph-mon[101299]: 1.0 scrub ok 2024-09-21T10:18:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:15 smithi045 ceph-mon[95976]: pgmap v862: 99 pgs: 10 unknown, 14 active+undersized, 11 active+undersized+degraded, 64 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 7476/89724 objects degraded (8.332%) 2024-09-21T10:18:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:15 smithi045 ceph-mon[95976]: 1.0 scrub starts 2024-09-21T10:18:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:15 smithi045 ceph-mon[95976]: 1.0 scrub ok 2024-09-21T10:18:16.037 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:16.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:16.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:16.176 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:16.216 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:16.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:16.317 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:16.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.354 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:16.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:16.454 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:16.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.498 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:16.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:16.519 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-09-21T10:18:16.519 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:18:16.519 INFO:tasks.thrashosds.thrasher:Reweighting osd 7 to 0.5035995256903933 2024-09-21T10:18:16.519 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 7 0.5035995256903933 2024-09-21T10:18:16.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:16.631 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:16.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:16.732 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:16.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.786 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:16.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:16.887 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:16.919 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:16.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:17.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:16 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 7476/89724 objects degraded (8.332%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:17.058 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:17.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:17.160 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:17.193 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:17.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:17.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:16 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 7476/89724 objects degraded (8.332%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:17.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:16 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 7476/89724 objects degraded (8.332%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:17.295 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:17.333 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:17.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:17.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:17.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.476 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:17.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:17.577 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:17.610 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:17.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:17.712 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:17.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:17.871 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.919 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:17.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:17.933 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:17 smithi032 ceph-mon[101299]: pgmap v863: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 989 op/s; 6888/81954 objects degraded (8.405%) 2024-09-21T10:18:17.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:17 smithi032 ceph-mon[101299]: 1.1 scrub starts 2024-09-21T10:18:17.934 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:17 smithi032 ceph-mon[101299]: 1.1 scrub ok 2024-09-21T10:18:17.934 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:17 smithi032 ceph-mon[105748]: pgmap v863: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 989 op/s; 6888/81954 objects degraded (8.405%) 2024-09-21T10:18:17.934 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:17 smithi032 ceph-mon[105748]: 1.1 scrub starts 2024-09-21T10:18:17.934 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:17 smithi032 ceph-mon[105748]: 1.1 scrub ok 2024-09-21T10:18:18.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:17 smithi045 ceph-mon[95976]: pgmap v863: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 989 op/s; 6888/81954 objects degraded (8.405%) 2024-09-21T10:18:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:17 smithi045 ceph-mon[95976]: 1.1 scrub starts 2024-09-21T10:18:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:17 smithi045 ceph-mon[95976]: 1.1 scrub ok 2024-09-21T10:18:18.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:18.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:18.174 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:18.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:18.323 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:18.353 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:18.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:18.455 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:18.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:18.603 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:18.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:18.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:18.743 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:18.769 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:18.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:18.871 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:18.897 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:18.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:18.958 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:18.999 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:19.046 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:19.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:19.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:19.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:19.116 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:19.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:19.252 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:19.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:19.294 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:19.354 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:19.385 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:19.487 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:19.520 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:19.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:19.622 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:19.653 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:19.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:19.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:18:19] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-21T10:18:19.755 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:19.775 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:18:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.799 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:19.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:19.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:19.940 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:19.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:20.034 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[105748]: pgmap v864: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 6888/81042 objects degraded (8.499%) 2024-09-21T10:18:20.034 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[105748]: 1.6 scrub starts 2024-09-21T10:18:20.034 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[105748]: 1.6 scrub ok 2024-09-21T10:18:20.034 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2261499088' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:20.034 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:20.034 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[101299]: pgmap v864: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 6888/81042 objects degraded (8.499%) 2024-09-21T10:18:20.035 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[101299]: 1.6 scrub starts 2024-09-21T10:18:20.035 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[101299]: 1.6 scrub ok 2024-09-21T10:18:20.035 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2261499088' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:20.035 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:19 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:19 smithi045 ceph-mon[95976]: pgmap v864: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 6888/81042 objects degraded (8.499%) 2024-09-21T10:18:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:19 smithi045 ceph-mon[95976]: 1.6 scrub starts 2024-09-21T10:18:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:19 smithi045 ceph-mon[95976]: 1.6 scrub ok 2024-09-21T10:18:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2261499088' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:19 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:20.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:20.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.073 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:20.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:20.173 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:20.199 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:20.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:20.300 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:20.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:20.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:20.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:20.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:20.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:20.579 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:20.615 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:20.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:20.717 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:20.751 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:20.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:20.853 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:20.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:20.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:20.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:21.035 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:21.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:21.136 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:21.171 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[105748]: 1.7 scrub starts 2024-09-21T10:18:21.172 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[105748]: 1.7 scrub ok 2024-09-21T10:18:21.172 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:18:21.172 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:21.172 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[105748]: osdmap e292: 8 total, 7 up, 8 in 2024-09-21T10:18:21.172 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[101299]: 1.7 scrub starts 2024-09-21T10:18:21.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[101299]: 1.7 scrub ok 2024-09-21T10:18:21.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:18:21.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:21.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:20 smithi032 ceph-mon[101299]: osdmap e292: 8 total, 7 up, 8 in 2024-09-21T10:18:21.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.176 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:21.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:21.277 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:20 smithi045 ceph-mon[95976]: 1.7 scrub starts 2024-09-21T10:18:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:20 smithi045 ceph-mon[95976]: 1.7 scrub ok 2024-09-21T10:18:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:20 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:18:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:20 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:20 smithi045 ceph-mon[95976]: osdmap e292: 8 total, 7 up, 8 in 2024-09-21T10:18:21.307 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:21.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:21.408 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:21.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:21.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:21.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:21.584 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:21.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:21.685 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:21.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.726 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:21.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:21.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:21.881 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:21.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:21.983 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:22.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.022 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:22.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:22.074 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:21 smithi032 ceph-mon[105748]: pgmap v866: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5726/68340 objects degraded (8.379%) 2024-09-21T10:18:22.074 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:21 smithi032 ceph-mon[105748]: 1.3 scrub starts 2024-09-21T10:18:22.074 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:21 smithi032 ceph-mon[105748]: 1.3 scrub ok 2024-09-21T10:18:22.074 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:21 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 5726/68340 objects degraded (8.379%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:22.074 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:21 smithi032 ceph-mon[101299]: pgmap v866: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5726/68340 objects degraded (8.379%) 2024-09-21T10:18:22.074 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:21 smithi032 ceph-mon[101299]: 1.3 scrub starts 2024-09-21T10:18:22.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:21 smithi032 ceph-mon[101299]: 1.3 scrub ok 2024-09-21T10:18:22.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:21 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 5726/68340 objects degraded (8.379%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:22.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:22.159 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:22.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:22.261 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:22.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:21 smithi045 ceph-mon[95976]: pgmap v866: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5726/68340 objects degraded (8.379%) 2024-09-21T10:18:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:21 smithi045 ceph-mon[95976]: 1.3 scrub starts 2024-09-21T10:18:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:21 smithi045 ceph-mon[95976]: 1.3 scrub ok 2024-09-21T10:18:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:21 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 5726/68340 objects degraded (8.379%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:22.298 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:22.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:22.400 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:22.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:22.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:22.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:22.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.592 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:22.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:22.693 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:22.720 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:22.822 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:22.852 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:22.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:22.954 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:22.990 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:23.011 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:18:23.077 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:22 smithi032 ceph-mon[105748]: 1.9 deep-scrub starts 2024-09-21T10:18:23.077 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:22 smithi032 ceph-mon[105748]: 1.9 deep-scrub ok 2024-09-21T10:18:23.077 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:22 smithi032 ceph-mon[101299]: 1.9 deep-scrub starts 2024-09-21T10:18:23.077 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:22 smithi032 ceph-mon[101299]: 1.9 deep-scrub ok 2024-09-21T10:18:23.093 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:23.118 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:23.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:23.220 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:23.254 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:23.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:23.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:22 smithi045 ceph-mon[95976]: 1.9 deep-scrub starts 2024-09-21T10:18:23.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:22 smithi045 ceph-mon[95976]: 1.9 deep-scrub ok 2024-09-21T10:18:23.355 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:23.382 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:23.485 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:23.523 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:23.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:23.625 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:23.655 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:23.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:23.756 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:23.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:23.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:23.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:23.941 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:23.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:24.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:24.072 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:24.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:24.133 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:23 smithi032 ceph-mon[105748]: pgmap v867: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 5726/67536 objects degraded (8.478%) 2024-09-21T10:18:24.134 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:23 smithi032 ceph-mon[105748]: 1.4 scrub starts 2024-09-21T10:18:24.134 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:23 smithi032 ceph-mon[105748]: 1.4 scrub ok 2024-09-21T10:18:24.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:23 smithi032 ceph-mon[101299]: pgmap v867: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 5726/67536 objects degraded (8.478%) 2024-09-21T10:18:24.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:23 smithi032 ceph-mon[101299]: 1.4 scrub starts 2024-09-21T10:18:24.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:23 smithi032 ceph-mon[101299]: 1.4 scrub ok 2024-09-21T10:18:24.175 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:24.220 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:24.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:23 smithi045 ceph-mon[95976]: pgmap v867: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 5726/67536 objects degraded (8.478%) 2024-09-21T10:18:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:23 smithi045 ceph-mon[95976]: 1.4 scrub starts 2024-09-21T10:18:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:23 smithi045 ceph-mon[95976]: 1.4 scrub ok 2024-09-21T10:18:24.327 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:24.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.388 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:24.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:24.489 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:24.533 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:24.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:24.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:24.667 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:24.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:24.769 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:24.812 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:24.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:24.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:24.966 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:24.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:25.068 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:25.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.117 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:25.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:25.184 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:24 smithi032 ceph-mon[101299]: 1.2 scrub starts 2024-09-21T10:18:25.184 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:24 smithi032 ceph-mon[101299]: 1.2 scrub ok 2024-09-21T10:18:25.184 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:24 smithi032 ceph-mon[105748]: 1.2 scrub starts 2024-09-21T10:18:25.185 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:24 smithi032 ceph-mon[105748]: 1.2 scrub ok 2024-09-21T10:18:25.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:25.271 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:24 smithi045 ceph-mon[95976]: 1.2 scrub starts 2024-09-21T10:18:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:24 smithi045 ceph-mon[95976]: 1.2 scrub ok 2024-09-21T10:18:25.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:25.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:25.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:25.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:25.538 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:25.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:25.639 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:25.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:25.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:25.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:25.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.816 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:25.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:25.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:18:25.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:18:25.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:18:25.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:18:25.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:18:25.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:18:25.860 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:18:25.860 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:18:25.860 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:25.861 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:25.861 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:25.861 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:25.861 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:18:25.861 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:25.861 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:25.918 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.948 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:25.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:26.049 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:26.074 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:26.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:26.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:26.203 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:26.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[105748]: pgmap v868: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 5112/62184 objects degraded (8.221%) 2024-09-21T10:18:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[105748]: 1.a scrub starts 2024-09-21T10:18:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[105748]: 1.a scrub ok 2024-09-21T10:18:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2674332604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:26.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[101299]: pgmap v868: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 5112/62184 objects degraded (8.221%) 2024-09-21T10:18:26.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[101299]: 1.a scrub starts 2024-09-21T10:18:26.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[101299]: 1.a scrub ok 2024-09-21T10:18:26.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2674332604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:26.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:26.211 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:25 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:25 smithi045 ceph-mon[95976]: pgmap v868: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 5112/62184 objects degraded (8.221%) 2024-09-21T10:18:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:25 smithi045 ceph-mon[95976]: 1.a scrub starts 2024-09-21T10:18:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:25 smithi045 ceph-mon[95976]: 1.a scrub ok 2024-09-21T10:18:26.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2674332604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:26.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:25 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:26.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:25 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:26.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:26.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:26.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:26.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:26.464 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:26.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:26.566 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:26.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:26.694 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:26.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.719 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:26.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:26.820 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:26.861 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:26.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:26.962 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:26.994 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:26.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:27.043 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:26 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 5112/62184 objects degraded (8.221%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:27.044 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:26 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 5112/62184 objects degraded (8.221%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:27.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:27.118 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:27.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:27.220 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:27.255 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:27.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:27.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:26 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 5112/62184 objects degraded (8.221%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:27.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:27.401 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:27.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:27.503 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:27.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.533 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:27.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:18:27.633 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:18:27.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:27.684 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:27.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:27.786 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:27.811 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:27.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:27.913 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:27.947 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:27.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:27 smithi032 ceph-mon[105748]: pgmap v869: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 918 op/s; 4446/53925 objects degraded (8.245%) 2024-09-21T10:18:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:27 smithi032 ceph-mon[101299]: pgmap v869: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 918 op/s; 4446/53925 objects degraded (8.245%) 2024-09-21T10:18:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:27 smithi045 ceph-mon[95976]: pgmap v869: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 918 op/s; 4446/53925 objects degraded (8.245%) 2024-09-21T10:18:28.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:28.091 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:28.193 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:28.252 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:18:28.252 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:18:28.258 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:28.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:28.360 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:28.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:28.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:28.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:28.554 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:28.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:28.655 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:28.682 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:28.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:28.783 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:28.828 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:28.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:28.877 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[105748]: 1.8 scrub starts 2024-09-21T10:18:28.877 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[105748]: 1.8 scrub ok 2024-09-21T10:18:28.877 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:18:28.877 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:18:28.877 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:28.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[101299]: 1.8 scrub starts 2024-09-21T10:18:28.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[101299]: 1.8 scrub ok 2024-09-21T10:18:28.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:18:28.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:18:28.878 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:28 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:28.930 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:28.971 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:28.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:29.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:29.098 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:29.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:29.200 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:29.231 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:29.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:28 smithi045 ceph-mon[95976]: 1.8 scrub starts 2024-09-21T10:18:29.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:28 smithi045 ceph-mon[95976]: 1.8 scrub ok 2024-09-21T10:18:29.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:28 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:18:29.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:28 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:18:29.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:28 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:29.332 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:29.358 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:29.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:29.369 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:18:29.461 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:29.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:29.606 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:18:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:18:29] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-21T10:18:29.619 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:29.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:18:29.634 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.679 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:29.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:29.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:29.836 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:29.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:29.888 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:29 smithi032 ceph-mon[101299]: pgmap v870: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 4446/53019 objects degraded (8.386%) 2024-09-21T10:18:29.939 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:29.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.999 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:29.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:30.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:30.135 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:29 smithi032 ceph-mon[105748]: pgmap v870: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 4446/53019 objects degraded (8.386%) 2024-09-21T10:18:30.141 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:30.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:30.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:30.277 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:30.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:29 smithi045 ceph-mon[95976]: pgmap v870: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 1.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 4446/53019 objects degraded (8.386%) 2024-09-21T10:18:30.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:30.410 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:30.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:30.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:30.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:30.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:30.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:30.721 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:30.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:30.823 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:30.861 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:30.889 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.7 to 0.5036 (80eb) 2024-09-21T10:18:30.963 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:30.994 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:31.096 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:31.118 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:31.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:31.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:30 smithi032 ceph-mon[101299]: 1.5 scrub starts 2024-09-21T10:18:31.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:30 smithi032 ceph-mon[101299]: 1.5 scrub ok 2024-09-21T10:18:31.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:30 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1013967115' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.5035995256903933} : dispatch 2024-09-21T10:18:31.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:30 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.5035995256903933} : dispatch 2024-09-21T10:18:31.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:30 smithi032 ceph-mon[105748]: 1.5 scrub starts 2024-09-21T10:18:31.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:30 smithi032 ceph-mon[105748]: 1.5 scrub ok 2024-09-21T10:18:31.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:30 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1013967115' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.5035995256903933} : dispatch 2024-09-21T10:18:31.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:30 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.5035995256903933} : dispatch 2024-09-21T10:18:31.220 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:31.242 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:31.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:30 smithi045 ceph-mon[95976]: 1.5 scrub starts 2024-09-21T10:18:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:30 smithi045 ceph-mon[95976]: 1.5 scrub ok 2024-09-21T10:18:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:30 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1013967115' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.5035995256903933} : dispatch 2024-09-21T10:18:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:30 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.5035995256903933} : dispatch 2024-09-21T10:18:31.344 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:31.365 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:31.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:31.467 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:31.489 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:31.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:31.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:31.613 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:31.715 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:31.738 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:31.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:31.840 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:31.862 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:31.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:31.964 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:31.991 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:31.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:32.092 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:32.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:32.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[101299]: pgmap v871: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 850 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 3331/40674 objects degraded (8.190%) 2024-09-21T10:18:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.5035995256903933}]': finished 2024-09-21T10:18:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[101299]: osdmap e293: 8 total, 7 up, 8 in 2024-09-21T10:18:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 3331/40674 objects degraded (8.190%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[101299]: osdmap e294: 8 total, 7 up, 8 in 2024-09-21T10:18:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[105748]: pgmap v871: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 850 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 3331/40674 objects degraded (8.190%) 2024-09-21T10:18:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.5035995256903933}]': finished 2024-09-21T10:18:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[105748]: osdmap e293: 8 total, 7 up, 8 in 2024-09-21T10:18:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 3331/40674 objects degraded (8.190%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:32.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:31 smithi032 ceph-mon[105748]: osdmap e294: 8 total, 7 up, 8 in 2024-09-21T10:18:32.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:32.273 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:32.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:31 smithi045 ceph-mon[95976]: pgmap v871: 99 pgs: 14 active+undersized, 11 active+undersized+degraded, 74 active+clean; 850 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 3331/40674 objects degraded (8.190%) 2024-09-21T10:18:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:31 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.5035995256903933}]': finished 2024-09-21T10:18:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:31 smithi045 ceph-mon[95976]: osdmap e293: 8 total, 7 up, 8 in 2024-09-21T10:18:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:31 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 3331/40674 objects degraded (8.190%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:31 smithi045 ceph-mon[95976]: osdmap e294: 8 total, 7 up, 8 in 2024-09-21T10:18:32.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:32.398 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:32.500 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:32.527 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:32.629 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:32.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:32.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:32.768 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:32.792 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:32.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:32.893 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:32.919 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:32.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:33.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:33.043 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:33.145 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:33.173 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:33.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:33.275 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:33.298 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:33.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:33.439 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:33.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:33.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:33.578 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:33.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:33.680 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:33.719 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:33.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:33.821 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:33.856 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[105748]: pgmap v874: 99 pgs: 2 peering, 14 active+undersized, 11 active+undersized+degraded, 72 active+clean; 835 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 3331/39960 objects degraded (8.336%) 2024-09-21T10:18:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[105748]: osdmap e295: 8 total, 7 up, 8 in 2024-09-21T10:18:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[105748]: 7.5 starting backfill to osd.6 from (0'0,0'0] MAX to 292'14732 2024-09-21T10:18:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3769296571' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:18:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[101299]: pgmap v874: 99 pgs: 2 peering, 14 active+undersized, 11 active+undersized+degraded, 72 active+clean; 835 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 3331/39960 objects degraded (8.336%) 2024-09-21T10:18:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[101299]: osdmap e295: 8 total, 7 up, 8 in 2024-09-21T10:18:33.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[101299]: 7.5 starting backfill to osd.6 from (0'0,0'0] MAX to 292'14732 2024-09-21T10:18:33.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3769296571' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:18:33.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:33.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:33.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:33.985 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:33.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:33 smithi045 ceph-mon[95976]: pgmap v874: 99 pgs: 2 peering, 14 active+undersized, 11 active+undersized+degraded, 72 active+clean; 835 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 3331/39960 objects degraded (8.336%) 2024-09-21T10:18:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:33 smithi045 ceph-mon[95976]: osdmap e295: 8 total, 7 up, 8 in 2024-09-21T10:18:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:33 smithi045 ceph-mon[95976]: 7.5 starting backfill to osd.6 from (0'0,0'0] MAX to 292'14732 2024-09-21T10:18:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:33 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3769296571' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:18:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:34.052 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:18:34.087 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:34.115 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:34.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:34.217 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:34.250 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:34.352 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:34.376 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:34.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:34.478 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:34.508 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:34.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:34.610 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:34.632 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:34.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:34.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:34.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:34.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:34.917 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:34.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:34 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3769296571' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:18:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:34 smithi032 ceph-mon[105748]: osdmap e296: 8 total, 7 up, 8 in 2024-09-21T10:18:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:34 smithi032 ceph-mon[105748]: osdmap e297: 8 total, 7 up, 8 in 2024-09-21T10:18:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:34 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3769296571' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:18:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:34 smithi032 ceph-mon[101299]: osdmap e296: 8 total, 7 up, 8 in 2024-09-21T10:18:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:34 smithi032 ceph-mon[101299]: osdmap e297: 8 total, 7 up, 8 in 2024-09-21T10:18:35.019 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:34 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3769296571' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:18:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:34 smithi045 ceph-mon[95976]: osdmap e296: 8 total, 7 up, 8 in 2024-09-21T10:18:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:34 smithi045 ceph-mon[95976]: osdmap e297: 8 total, 7 up, 8 in 2024-09-21T10:18:35.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:35.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:35.149 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:35.171 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:35.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:35.273 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:35.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:35.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:35.422 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:35.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.454 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:35.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:35.555 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:35.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.596 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:35.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:35.686 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:35.691 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:35.696 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:35.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:35.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:35.823 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:35.852 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:35.953 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:35.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:35 smithi032 ceph-mon[105748]: pgmap v877: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 3 active+clean+remapped, 2 peering, 14 active+undersized, 10 active+undersized+degraded, 68 active+clean; 759 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 2976/36306 objects degraded (8.197%); 558/36306 objects misplaced (1.537%); 2.0 MiB/s, 34 objects/s recovering 2024-09-21T10:18:35.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:35 smithi032 ceph-mon[101299]: pgmap v877: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 3 active+clean+remapped, 2 peering, 14 active+undersized, 10 active+undersized+degraded, 68 active+clean; 759 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 2976/36306 objects degraded (8.197%); 558/36306 objects misplaced (1.537%); 2.0 MiB/s, 34 objects/s recovering 2024-09-21T10:18:35.983 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:35 smithi045 ceph-mon[95976]: pgmap v877: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 3 active+clean+remapped, 2 peering, 14 active+undersized, 10 active+undersized+degraded, 68 active+clean; 759 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 2976/36306 objects degraded (8.197%); 558/36306 objects misplaced (1.537%); 2.0 MiB/s, 34 objects/s recovering 2024-09-21T10:18:36.085 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:36.121 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:36.224 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:36.259 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:36.361 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:36.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.402 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:36.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:36.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:36.552 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:36.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:36.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:36.683 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:36.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:36 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 2976/36306 objects degraded (8.197%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:36.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:36 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 2976/36306 objects degraded (8.197%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:36.784 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:36.815 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:36.917 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:36.929 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-09-21T10:18:36.929 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:18:36.929 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:18:36.929 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:18:36.965 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:36.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:37.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:36 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 2976/36306 objects degraded (8.197%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:18:37.067 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.098 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:37.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:37.199 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:37.248 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:37.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:37.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:37.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:37.480 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:37.506 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:37.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:37.608 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:37.635 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:37.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:37.737 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:37.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:37.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:37.866 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:37.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:37.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:37.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:37 smithi032 ceph-mon[105748]: pgmap v879: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 11 active+clean+remapped, 2 peering, 11 active+undersized, 10 active+undersized+degraded, 63 active+clean; 665 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 171 KiB/s rd, 503 KiB/s wr, 691 op/s; 2594/31779 objects degraded (8.163%); 561/31779 objects misplaced (1.765%); 2.1 MiB/s, 31 objects/s recovering 2024-09-21T10:18:37.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:37 smithi032 ceph-mon[101299]: pgmap v879: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 11 active+clean+remapped, 2 peering, 11 active+undersized, 10 active+undersized+degraded, 63 active+clean; 665 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 171 KiB/s rd, 503 KiB/s wr, 691 op/s; 2594/31779 objects degraded (8.163%); 561/31779 objects misplaced (1.765%); 2.1 MiB/s, 31 objects/s recovering 2024-09-21T10:18:37.998 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:38.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:38.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:37 smithi045 ceph-mon[95976]: pgmap v879: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 11 active+clean+remapped, 2 peering, 11 active+undersized, 10 active+undersized+degraded, 63 active+clean; 665 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 171 KiB/s rd, 503 KiB/s wr, 691 op/s; 2594/31779 objects degraded (8.163%); 561/31779 objects misplaced (1.765%); 2.1 MiB/s, 31 objects/s recovering 2024-09-21T10:18:38.124 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:38.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:38.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:38.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:38.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:38.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:38.374 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:38.397 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:38.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:38.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:18:38.499 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:38.548 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:38.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:38.650 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:38.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.674 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:38.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:38.775 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:38.799 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:38.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:38.903 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:38.936 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:38.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:38.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:38 smithi032 ceph-mon[101299]: osdmap e298: 8 total, 7 up, 8 in 2024-09-21T10:18:38.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:38 smithi032 ceph-mon[105748]: osdmap e298: 8 total, 7 up, 8 in 2024-09-21T10:18:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:38 smithi045 ceph-mon[95976]: osdmap e298: 8 total, 7 up, 8 in 2024-09-21T10:18:39.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:39.064 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:39.166 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:39.192 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:39.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:39.294 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:39.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:39.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:39.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.453 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:39.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:39.554 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.596 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:39.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:39.697 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:39.725 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:39.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:39.826 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:39.856 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:39.875 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:18:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:18:39] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.51.0" 2024-09-21T10:18:39.875 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:39 smithi032 ceph-mon[105748]: pgmap v881: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 64 active+clean; 651 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 142 KiB/s rd, 416 KiB/s wr, 609 op/s; 2594/31125 objects degraded (8.334%); 1104/31125 objects misplaced (3.547%); 1.8 MiB/s, 25 objects/s recovering 2024-09-21T10:18:39.875 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:39 smithi032 ceph-mon[105748]: osdmap e299: 8 total, 7 up, 8 in 2024-09-21T10:18:39.875 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:39 smithi032 ceph-mon[101299]: pgmap v881: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 64 active+clean; 651 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 142 KiB/s rd, 416 KiB/s wr, 609 op/s; 2594/31125 objects degraded (8.334%); 1104/31125 objects misplaced (3.547%); 1.8 MiB/s, 25 objects/s recovering 2024-09-21T10:18:39.875 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:39 smithi032 ceph-mon[101299]: osdmap e299: 8 total, 7 up, 8 in 2024-09-21T10:18:39.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.010 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:40.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:39 smithi045 ceph-mon[95976]: pgmap v881: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 64 active+clean; 651 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 142 KiB/s rd, 416 KiB/s wr, 609 op/s; 2594/31125 objects degraded (8.334%); 1104/31125 objects misplaced (3.547%); 1.8 MiB/s, 25 objects/s recovering 2024-09-21T10:18:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:39 smithi045 ceph-mon[95976]: osdmap e299: 8 total, 7 up, 8 in 2024-09-21T10:18:40.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:40.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:40.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:40.250 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.278 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:40.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.380 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.406 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.535 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.637 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:40.665 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:40.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:40.767 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.792 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:40.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.893 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:40.920 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:41.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:41.068 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:41.169 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:41.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.214 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:41.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:41.268 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:41.315 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:41.344 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:18:41.345 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:18:41.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:41.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:41.378 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:18:41.378 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:41.422 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:18:41.422 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:41.449 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:41.462 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:18:41.462 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:41.496 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:41.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:41.513 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:18:41.513 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:41.514 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:18:41.554 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:41.568 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:18:41.569 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:41.588 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:18:41.588 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:18:41.588 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2414 7.b 7:dfc0d3f8:::benchmark_data_smithi032_200324_object2413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.533176+0000", 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 237.050318785, 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2667221340000001, 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.589 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2414 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.533176+0000", 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.590 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.533176+0000", 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.533178+0000", 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5400000000000001e-06 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.591 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.533200+0000", 2024-09-21T10:18:41.592 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1875999999999999e-05 2024-09-21T10:18:41.592 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.592 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.592 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.533210+0000", 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0498e-05 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.533217+0000", 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7440000000000001e-06 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.593 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.536645+0000", 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003428108 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.536656+0000", 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1308000000000001e-05 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.525703+0000", 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.98904711099999998 2024-09-21T10:18:41.594 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.525713+0000", 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8349999999999999e-06 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.797409+0000", 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2716959000000001 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.797550+0000", 2024-09-21T10:18:41.595 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014041499999999999 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.797713+0000", 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000163437 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799073+0000", 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0013597030000000001 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799253+0000", 2024-09-21T10:18:41.596 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018048700000000001 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799578+0000", 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000325073 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799726+0000", 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014767900000000001 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.597 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799898+0000", 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017242000000000001 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2432 7.b 7:d9ad3728:::benchmark_data_smithi032_200324_object2431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.542476+0000", 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 237.041019457, 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2582090250000002, 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.598 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2432 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.542476+0000", 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.542476+0000", 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.599 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.542478+0000", 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9080000000000002e-06 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.542500+0000", 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2357999999999998e-05 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.542517+0000", 2024-09-21T10:18:41.600 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6986e-05 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.542525+0000", 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7139999999999993e-06 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.542600+0000", 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5123999999999994e-05 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:18:41.601 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.542610+0000", 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.007e-05 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.525759+0000", 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.98314902800000004 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.525768+0000", 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5640000000000002e-06 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.602 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798416+0000", 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.272647139 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798565+0000", 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014973499999999999 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:18:41.603 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798726+0000", 2024-09-21T10:18:41.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000160877 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799285+0000", 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00055924200000000005 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800224+0000", 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00093877900000000005 2024-09-21T10:18:41.605 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800464+0000", 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024002600000000001 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800525+0000", 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0869999999999998e-05 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800685+0000", 2024-09-21T10:18:41.606 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000159605 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2406 7.0 7:0b6d93fa:::benchmark_data_smithi032_200324_object2405:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e253)", 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:45.522115+0000", 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 236.06138035399999, 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.278766581, 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:18:41.607 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2406 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522115+0000", 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522115+0000", 2024-09-21T10:18:41.608 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522116+0000", 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6220000000000001e-06 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522155+0000", 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8803000000000001e-05 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522169+0000", 2024-09-21T10:18:41.609 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4124000000000001e-05 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522180+0000", 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0856000000000001e-05 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522278+0000", 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7695000000000006e-05 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.610 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op must wait for map", 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522285+0000", 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4410000000000004e-06 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.523461+0000", 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0011754000000000001 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.523469+0000", 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5779999999999999e-06 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.611 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.797393+0000", 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2739238799999999 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for degraded object", 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798038+0000", 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00064500400000000002 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800788+0000", 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0027502640000000001 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.612 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800881+0000", 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2913999999999998e-05 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2427 7.0 7:06f9fafc:::benchmark_data_smithi032_200324_object2426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e253)", 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:45.522201+0000", 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 236.061294586, 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.279038111, 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.613 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2427 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522201+0000", 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522201+0000", 2024-09-21T10:18:41.614 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522201+0000", 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0399999999999996e-07 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522263+0000", 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1749999999999997e-05 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522274+0000", 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0654e-05 2024-09-21T10:18:41.615 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522281+0000", 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9859999999999997e-06 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522367+0000", 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6755999999999994e-05 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522373+0000", 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8919999999999997e-06 2024-09-21T10:18:41.616 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.523777+0000", 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014040210000000001 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.523784+0000", 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7580000000000003e-06 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798407+0000", 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2746234860000001 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.617 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798563+0000", 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015532999999999999 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798736+0000", 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000173046 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799914+0000", 2024-09-21T10:18:41.618 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001177937 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800740+0000", 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00082580900000000005 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801039+0000", 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029921899999999998 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.619 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801095+0000", 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6648999999999999e-05 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801239+0000", 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014321399999999999 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2429 7.0 7:0cd392e0:::benchmark_data_smithi032_200324_object2428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e253)", 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:45.522298+0000", 2024-09-21T10:18:41.620 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 236.06119747400001, 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.279199371, 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2429 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522298+0000", 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.621 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522298+0000", 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522298+0000", 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9800000000000003e-07 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522358+0000", 2024-09-21T10:18:41.622 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0270999999999999e-05 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522369+0000", 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0389e-05 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522375+0000", 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5830000000000003e-06 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.623 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522488+0000", 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000112908 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.522494+0000", 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4670000000000003e-06 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524832+0000", 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0023382540000000001 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:18:41.624 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524840+0000", 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9999999999999996e-06 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799124+0000", 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2742841739999999 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799307+0000", 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018242700000000001 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:18:41.625 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799510+0000", 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00020288899999999999 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800261+0000", 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00075113299999999997 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800921+0000", 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00065992100000000005 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.626 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801283+0000", 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000362161 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801344+0000", 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1598000000000002e-05 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801497+0000", 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015259799999999999 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.627 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244548 7.9 7:91100c2f:::benchmark_data_smithi032_200324_object119882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.401174+0000", 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.182321343, 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.025668159999999999, 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244548 2024-09-21T10:18:41.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.401174+0000", 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.401174+0000", 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.401175+0000", 2024-09-21T10:18:41.629 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5859999999999999e-06 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.401182+0000", 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3949999999999998e-06 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.401201+0000", 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9120999999999999e-05 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.401211+0000", 2024-09-21T10:18:41.630 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6930000000000002e-06 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.401271+0000", 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0652999999999997e-05 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.401399+0000", 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000127587 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:18:41.631 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.401521+0000", 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012223799999999999 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.404631+0000", 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003109784 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.405504+0000", 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00087321600000000001 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.632 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.426700+0000", 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.021195538 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.426793+0000", 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3438999999999998e-05 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.426842+0000", 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8909999999999998e-05 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.633 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244552 7.9 7:9fa2744b:::benchmark_data_smithi032_200324_object119886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.406881+0000", 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.17661410499999999, 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.028775901999999999, 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.634 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244552 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.406881+0000", 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.406881+0000", 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.635 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.406882+0000", 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3090000000000001e-06 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.406888+0000", 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6690000000000003e-06 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.406902+0000", 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4273e-05 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.636 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.406910+0000", 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4270000000000002e-06 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.406976+0000", 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5862000000000001e-05 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.407090+0000", 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114067 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.637 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.407203+0000", 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000113286 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.426888+0000", 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.019685023999999999 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.426923+0000", 2024-09-21T10:18:41.638 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4934000000000001e-05 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.435513+0000", 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0085903349999999993 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.435613+0000", 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000100259 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.435657+0000", 2024-09-21T10:18:41.639 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3457000000000001e-05 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244560 7.9 7:980ba33c:::benchmark_data_smithi032_200324_object119894:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.409984+0000", 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.173511153, 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.025922376, 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.640 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244560 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.409984+0000", 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.409984+0000", 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.409986+0000", 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.663e-06 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.409991+0000", 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5049999999999998e-06 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.410008+0000", 2024-09-21T10:18:41.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6769000000000002e-05 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.410016+0000", 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1459999999999996e-06 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.416696+0000", 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0066801869999999998 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.643 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.416867+0000", 2024-09-21T10:18:41.644 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017060099999999999 2024-09-21T10:18:41.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:18:41.644 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.417031+0000", 2024-09-21T10:18:41.644 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000163802 2024-09-21T10:18:41.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.645 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:18:41.645 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:41.645 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.426958+0000", 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0099275059999999991 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.426997+0000", 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8794000000000002e-05 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.435717+0000", 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0087201970000000007 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.435864+0000", 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014650199999999999 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.435906+0000", 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2704000000000003e-05 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.647 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:244554 7.5 e299/298 7:ad96c3f0:::benchmark_data_smithi032_200324_object119888:head v 299'15199, mlcod=299'15197)", 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.411713+0000", 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.17178242399999999, 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.019485853000000001, 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.411713+0000", 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.648 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.411713+0000", 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.411713+0000", 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0100000000000004e-07 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.411722+0000", 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0809999999999994e-06 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.411737+0000", 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5617e-05 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.411744+0000", 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9110000000000004e-06 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.411862+0000", 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000118007 2024-09-21T10:18:41.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.411883+0000", 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0871999999999999e-05 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.431156+0000", 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.019273288999999999 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.431199+0000", 2024-09-21T10:18:41.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2274999999999998e-05 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244564 7.9 7:9958a430:::benchmark_data_smithi032_200324_object119898:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.414367+0000", 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.16912782100000001, 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.021722201, 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.652 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244564 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.414367+0000", 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.414367+0000", 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.653 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.414369+0000", 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3489999999999999e-06 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.414373+0000", 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6650000000000002e-06 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.414385+0000", 2024-09-21T10:18:41.654 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2094e-05 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.414393+0000", 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5179999999999998e-06 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.427029+0000", 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012635983 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.655 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.427153+0000", 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012392800000000001 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.427278+0000", 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012471800000000001 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.435705+0000", 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.008427482 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.435917+0000", 2024-09-21T10:18:41.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00021230399999999999 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.435958+0000", 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0231000000000002e-05 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.436049+0000", 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1658999999999998e-05 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.436089+0000", 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0269999999999999e-05 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244567 7.9 7:95744442:::benchmark_data_smithi032_200324_object119901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.418935+0000", 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.16455988599999999, 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.017713808000000001, 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.658 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244567 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.418935+0000", 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.418935+0000", 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.418936+0000", 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2500000000000001e-06 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.418942+0000", 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2129999999999999e-06 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.418960+0000", 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8247999999999999e-05 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.418967+0000", 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.503e-06 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.430628+0000", 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011660780000000001 2024-09-21T10:18:41.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.430753+0000", 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012445800000000001 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.430875+0000", 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012261300000000001 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.436133+0000", 2024-09-21T10:18:41.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00525809 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.436415+0000", 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000281334 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.436510+0000", 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4974000000000002e-05 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.436571+0000", 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1229999999999995e-05 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.436649+0000", 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8114999999999993e-05 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244591 7.0 7:01c74eaa:::benchmark_data_smithi032_200324_object119925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.446297+0000", 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.13719801200000001, 2024-09-21T10:18:41.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.025765296, 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244591 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.446297+0000", 2024-09-21T10:18:41.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.446297+0000", 2024-09-21T10:18:41.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.667 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.446299+0000", 2024-09-21T10:18:41.667 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6330000000000001e-06 2024-09-21T10:18:41.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.667 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.446305+0000", 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0929999999999999e-06 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.448824+0000", 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002519486 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.448842+0000", 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7266e-05 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.668 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.448963+0000", 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000121323 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.449096+0000", 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000132824 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3", 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.449222+0000", 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012586699999999999 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.452787+0000", 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0035655610000000001 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.471867+0000", 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.019080202000000001 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.471968+0000", 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000100362 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.472062+0000", 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4679000000000006e-05 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244592 7.0 7:0faf7596:::benchmark_data_smithi032_200324_object119926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.448882+0000", 2024-09-21T10:18:41.671 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.13461341399999999, 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.023376734, 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244592 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.448882+0000", 2024-09-21T10:18:41.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.448882+0000", 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.448883+0000", 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2139999999999999e-06 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.673 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.448888+0000", 2024-09-21T10:18:41.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7890000000000002e-06 2024-09-21T10:18:41.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.674 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.448899+0000", 2024-09-21T10:18:41.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1809e-05 2024-09-21T10:18:41.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.448906+0000", 2024-09-21T10:18:41.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4479999999999999e-06 2024-09-21T10:18:41.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.452813+0000", 2024-09-21T10:18:41.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0039069899999999999 2024-09-21T10:18:41.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.452952+0000", 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013889999999999999 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3", 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.453089+0000", 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013673400000000001 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.457387+0000", 2024-09-21T10:18:41.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0042987260000000001 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.472100+0000", 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.014713192999999999 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.472180+0000", 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0003e-05 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.472258+0000", 2024-09-21T10:18:41.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7928000000000003e-05 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:244588 7.5 e299/298 7:a4aede53:::benchmark_data_smithi032_200324_object119922:head v 299'15206, mlcod=299'15201)", 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.458126+0000", 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.12536934599999999, 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.016941580000000001, 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.679 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458126+0000", 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458126+0000", 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.680 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458127+0000", 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0979999999999999e-06 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458134+0000", 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8419999999999999e-06 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458149+0000", 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5579000000000001e-05 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.681 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458157+0000", 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6899999999999992e-06 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458326+0000", 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016930299999999999 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458356+0000", 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0162e-05 2024-09-21T10:18:41.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.474981+0000", 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.016624338999999998 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.475067+0000", 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6567e-05 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:244590 7.5 e299/298 7:af321ecf:::benchmark_data_smithi032_200324_object119924:head v 299'15207, mlcod=299'15201)", 2024-09-21T10:18:41.683 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.458197+0000", 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.12529821899999999, 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.016932163, 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458197+0000", 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458197+0000", 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458197+0000", 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1699999999999999e-07 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458201+0000", 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9349999999999996e-06 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458209+0000", 2024-09-21T10:18:41.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3359999999999997e-06 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.458214+0000", 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5219999999999998e-06 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.462294+0000", 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0040795850000000002 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.462314+0000", 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0298000000000001e-05 2024-09-21T10:18:41.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.475073+0000", 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012758709999999999 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.475129+0000", 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6360000000000002e-05 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.687 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244615 7.0 7:0e599a6a:::benchmark_data_smithi032_200324_object119949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.481188+0000", 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.10230750399999999, 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.021307959000000001, 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244615 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.688 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481188+0000", 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481188+0000", 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481188+0000", 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8400000000000003e-07 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481193+0000", 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6949999999999999e-06 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481205+0000", 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1402e-05 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481212+0000", 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3830000000000003e-06 2024-09-21T10:18:41.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.485072+0000", 2024-09-21T10:18:41.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0038596949999999998 2024-09-21T10:18:41.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.490107+0000", 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0050354329999999997 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3", 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.490239+0000", 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000131457 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.694 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.502345+0000", 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.012106541 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.502359+0000", 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4037e-05 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.502405+0000", 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5453e-05 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.502496+0000", 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0978999999999997e-05 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:244610 7.4 e299/248 7:2cc63ed3:::benchmark_data_smithi032_200324_object119944:head v 299'15196, mlcod=299'15194)", 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.481497+0000", 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.10199839600000001, 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.017981942000000001, 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.696 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481497+0000", 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481497+0000", 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481498+0000", 2024-09-21T10:18:41.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6800000000000009e-07 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481505+0000", 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0230000000000004e-06 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481515+0000", 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0134e-05 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.698 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.481522+0000", 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4719999999999998e-06 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.485434+0000", 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0039120420000000001 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.485457+0000", 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2424000000000001e-05 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.699 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.499401+0000", 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013943804000000001 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.499479+0000", 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8075e-05 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244620 7.0 7:01d1d482:::benchmark_data_smithi032_200324_object119954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.484529+0000", 2024-09-21T10:18:41.700 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.098966281000000003, 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.018128954999999999, 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244620 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.484529+0000", 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.701 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.484529+0000", 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.484531+0000", 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9309999999999998e-06 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.484537+0000", 2024-09-21T10:18:41.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5200000000000003e-06 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.484551+0000", 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3273e-05 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.484561+0000", 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.059e-05 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.495111+0000", 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010550131000000001 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.495244+0000", 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013315199999999999 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3", 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.495381+0000", 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013685499999999999 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.502503+0000", 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0071215790000000003 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.502536+0000", 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.3522999999999997e-05 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.502578+0000", 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1526999999999997e-05 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.705 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.502658+0000", 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9874000000000001e-05 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244624 7.0 7:043b7434:::benchmark_data_smithi032_200324_object119958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.495001+0000", 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.088493894000000003, 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.023892252999999999, 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.706 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244624 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.495001+0000", 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.495001+0000", 2024-09-21T10:18:41.707 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.495003+0000", 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8139999999999999e-06 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.495007+0000", 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6990000000000001e-06 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.495016+0000", 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7399999999999999e-06 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.495023+0000", 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.951e-06 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.499197+0000", 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0041739840000000004 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.709 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.499417+0000", 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000219772 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3", 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.499552+0000", 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013491299999999999 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.509141+0000", 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0095887690000000005 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.517057+0000", 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0079158160000000009 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.517116+0000", 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9385999999999997e-05 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.518893+0000", 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001777409 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.711 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45206.0:244630 7.2 e299/245 7:455e0ed2:::benchmark_data_smithi032_200324_object119964:head v 299'15166, mlcod=299'15163)", 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.504645+0000", 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.078850562999999999, 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.017196118, 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.504645+0000", 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.504645+0000", 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.504645+0000", 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5100000000000001e-07 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.504647+0000", 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6790000000000001e-06 2024-09-21T10:18:41.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.504653+0000", 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8850000000000004e-06 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.504658+0000", 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3159999999999998e-06 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.508193+0000", 2024-09-21T10:18:41.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0035348810000000001 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.508211+0000", 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8680000000000001e-05 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.521757+0000", 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013545771 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:18:41.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.521841+0000", 2024-09-21T10:18:41.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3654999999999998e-05 2024-09-21T10:18:41.716 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.716 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.716 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.716 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:18:41.716 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:18:41.716 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:18:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:41.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:41.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:41.857 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:41.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:41 smithi032 ceph-mon[105748]: pgmap v883: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 430 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 137 KiB/s rd, 403 KiB/s wr, 976 op/s; 1681/20508 objects degraded (8.197%); 548/20508 objects misplaced (2.672%); 5.5 MiB/s, 84 objects/s recovering 2024-09-21T10:18:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:41 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3677947672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:41 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 1681/20508 objects degraded (8.197%), 10 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:41 smithi032 ceph-mon[101299]: pgmap v883: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 430 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 137 KiB/s rd, 403 KiB/s wr, 976 op/s; 1681/20508 objects degraded (8.197%); 548/20508 objects misplaced (2.672%); 5.5 MiB/s, 84 objects/s recovering 2024-09-21T10:18:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:41 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3677947672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:41 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 1681/20508 objects degraded (8.197%), 10 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:41.959 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.010 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:42.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:41 smithi045 ceph-mon[95976]: pgmap v883: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 430 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 137 KiB/s rd, 403 KiB/s wr, 976 op/s; 1681/20508 objects degraded (8.197%); 548/20508 objects misplaced (2.672%); 5.5 MiB/s, 84 objects/s recovering 2024-09-21T10:18:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:41 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3677947672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:41 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 1681/20508 objects degraded (8.197%), 10 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:42.112 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:42.145 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:42.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:42.247 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.291 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:42.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:42.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:42.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.428 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:42.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:42.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:42.573 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:42.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:42.675 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:42.714 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:42.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:42.816 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:42.854 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:42.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:42.956 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:42.992 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:42.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:43.094 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:43.130 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:43.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:43.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:43.256 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:43.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:43.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:43.395 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:18:43.406 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:43.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:43.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:43.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:43.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:43.698 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:43.800 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:43.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.831 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:43.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:43.918 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:43 smithi032 ceph-mon[105748]: pgmap v884: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 416 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 105 KiB/s rd, 310 KiB/s wr, 780 op/s; 1681/19827 objects degraded (8.478%); 321/19827 objects misplaced (1.619%); 4.2 MiB/s, 64 objects/s recovering 2024-09-21T10:18:43.919 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:43 smithi032 ceph-mon[101299]: pgmap v884: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 416 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 105 KiB/s rd, 310 KiB/s wr, 780 op/s; 1681/19827 objects degraded (8.478%); 321/19827 objects misplaced (1.619%); 4.2 MiB/s, 64 objects/s recovering 2024-09-21T10:18:43.931 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:43.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.972 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:43.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:43 smithi045 ceph-mon[95976]: pgmap v884: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 416 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 105 KiB/s rd, 310 KiB/s wr, 780 op/s; 1681/19827 objects degraded (8.478%); 321/19827 objects misplaced (1.619%); 4.2 MiB/s, 64 objects/s recovering 2024-09-21T10:18:44.073 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:44.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.117 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:44.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:44.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:44.254 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:44.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:44.356 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:44.407 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:44.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:44.545 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:44.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:44.646 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:44.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:44.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:44.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:44.820 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:44.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:44.922 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:44.951 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:44.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:45.053 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:45.093 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:45.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:45.194 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:45.210 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:18:45.211 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:18:45.223 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:45.324 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:45.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:45.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:45.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.514 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:45.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:45.614 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:45.646 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:45.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:45.748 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:45.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.805 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:45.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:45.906 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:45.937 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:45.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:45 smithi032 ceph-mon[101299]: pgmap v885: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 283 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 1036/13449 objects degraded (7.703%); 321/13449 objects misplaced (2.387%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T10:18:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:45 smithi032 ceph-mon[105748]: pgmap v885: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 283 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 1036/13449 objects degraded (7.703%); 321/13449 objects misplaced (2.387%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T10:18:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:45 smithi045 ceph-mon[95976]: pgmap v885: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 283 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 1036/13449 objects degraded (7.703%); 321/13449 objects misplaced (2.387%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T10:18:46.039 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:46.083 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:46.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:46.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:46.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:46.275 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Removed 124663 objects 2024-09-21T10:18:46.275 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Clean up completed and total clean up time :152.408 2024-09-21T10:18:46.279 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:18:46.281 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:18:46.328 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:46.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:46.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:46.452 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:46.475 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:46.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:46.577 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:46.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.602 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:46.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:46.703 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:46.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:46.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:46.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:46.851 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:46.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:46.953 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:46.975 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:46.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:46 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 1036/13449 objects degraded (7.703%), 10 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:47.077 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:47.101 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:47.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:47.132 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:47.202 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:46 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 1036/13449 objects degraded (7.703%), 10 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:46 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 1036/13449 objects degraded (7.703%), 10 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:47.211 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:18:47.224 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:47.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:47.327 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:47.349 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:47.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:47.451 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:47.539 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:47.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:47.564 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":885,"stamp":"2024-09-21T10:18:44.435983+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":296687613,"num_objects":4483,"num_object_clones":0,"num_object_copies":13449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1036,"num_objects_misplaced":321,"num_objects_unfound":0,"num_objects_dirty":4483,"num_whiteouts":0,"num_read":6376,"num_read_kb":13833,"num_write":245996,"num_write_kb":7993474,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14965,"num_bytes_recovered":986254295,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111262,"ondisk_log_size":111262,"up":263,"acting":276,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5995256,"kb_used_data":2256604,"kb_used_omap":35,"kb_used_meta":3738460,"kb_avail":743965960,"statfs":{"total":767960285184,"available":761821143040,"internally_reserved":0,"allocated":2310762496,"data_stored":2299675802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36146,"internal_metadata":3828183758},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-400424960,"num_objects":-6110,"num_object_clones":0,"num_object_copies":-18330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1558,"num_objects_misplaced":-240,"num_objects_unfound":0,"num_objects_dirty":-6110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6110,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":493,"num_bytes_recovered":32047104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002959"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":473,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.689176+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:18:38.689176+0000","last_peered":"2024-09-21T10:18:38.689176+0000","last_clean":"2024-09-21T10:18:38.689176+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:18:38.689176+0000","last_undegraded":"2024-09-21T10:18:38.689176+0000","last_fullsized":"2024-09-21T10:18:38.689176+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":270,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.686873+0000","last_change":"2024-09-21T10:03:19.765530+0000","last_active":"2024-09-21T10:18:37.686873+0000","last_peered":"2024-09-21T10:18:37.686873+0000","last_clean":"2024-09-21T10:18:37.686873+0000","last_became_active":"2024-09-21T10:03:19.764905+0000","last_became_peered":"2024-09-21T10:03:19.764905+0000","last_unstale":"2024-09-21T10:18:37.686873+0000","last_undegraded":"2024-09-21T10:18:37.686873+0000","last_fullsized":"2024-09-21T10:18:37.686873+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:53.360396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":191,"reported_epoch":298,"state":"active+undersized","last_fresh":"2024-09-21T10:18:37.687130+0000","last_change":"2024-09-21T10:17:35.175828+0000","last_active":"2024-09-21T10:18:37.687130+0000","last_peered":"2024-09-21T10:18:37.687130+0000","last_clean":"2024-09-21T10:03:19.759615+0000","last_became_active":"2024-09-21T10:17:35.175828+0000","last_became_peered":"2024-09-21T10:17:35.175828+0000","last_unstale":"2024-09-21T10:18:37.687130+0000","last_undegraded":"2024-09-21T10:18:37.687130+0000","last_fullsized":"2024-09-21T10:17:35.122679+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:34.208210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":130,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.689061+0000","last_change":"2024-09-21T10:18:31.643922+0000","last_active":"2024-09-21T10:18:38.689061+0000","last_peered":"2024-09-21T10:18:38.689061+0000","last_clean":"2024-09-21T10:18:38.689061+0000","last_became_active":"2024-09-21T10:18:31.643218+0000","last_became_peered":"2024-09-21T10:18:31.643218+0000","last_unstale":"2024-09-21T10:18:38.689061+0000","last_undegraded":"2024-09-21T10:18:38.689061+0000","last_fullsized":"2024-09-21T10:18:38.689061+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:33:16.266657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":307,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.689590+0000","last_change":"2024-09-21T10:18:31.639132+0000","last_active":"2024-09-21T10:18:38.689590+0000","last_peered":"2024-09-21T10:18:38.689590+0000","last_clean":"2024-09-21T10:18:38.689590+0000","last_became_active":"2024-09-21T10:18:31.638798+0000","last_became_peered":"2024-09-21T10:18:31.638798+0000","last_unstale":"2024-09-21T10:18:38.689590+0000","last_undegraded":"2024-09-21T10:18:38.689590+0000","last_fullsized":"2024-09-21T10:18:38.689590+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:17:19.036316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":253,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.686732+0000","last_change":"2024-09-21T09:50:22.399139+0000","last_active":"2024-09-21T10:18:37.686732+0000","last_peered":"2024-09-21T10:18:37.686732+0000","last_clean":"2024-09-21T10:18:37.686732+0000","last_became_active":"2024-09-21T09:50:22.398871+0000","last_became_peered":"2024-09-21T09:50:22.398871+0000","last_unstale":"2024-09-21T10:18:37.686732+0000","last_undegraded":"2024-09-21T10:18:37.686732+0000","last_fullsized":"2024-09-21T10:18:37.686732+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:30.247770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":487,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.689714+0000","last_change":"2024-09-21T10:18:31.642565+0000","last_active":"2024-09-21T10:18:38.689714+0000","last_peered":"2024-09-21T10:18:38.689714+0000","last_clean":"2024-09-21T10:18:38.689714+0000","last_became_active":"2024-09-21T10:18:31.641900+0000","last_became_peered":"2024-09-21T10:18:31.641900+0000","last_unstale":"2024-09-21T10:18:38.689714+0000","last_undegraded":"2024-09-21T10:18:38.689714+0000","last_fullsized":"2024-09-21T10:18:38.689714+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:39.165018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":324,"reported_epoch":298,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:37.686067+0000","last_change":"2024-09-21T10:17:35.124435+0000","last_active":"2024-09-21T10:18:37.686067+0000","last_peered":"2024-09-21T10:18:37.686067+0000","last_clean":"2024-09-21T10:17:25.108942+0000","last_became_active":"2024-09-21T10:17:35.124435+0000","last_became_peered":"2024-09-21T10:17:35.124435+0000","last_unstale":"2024-09-21T10:18:37.686067+0000","last_undegraded":"2024-09-21T10:17:35.121018+0000","last_fullsized":"2024-09-21T10:17:35.120899+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:44.888283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"299'15297","reported_seq":15708,"reported_epoch":299,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:40.166323+0000","last_change":"2024-09-21T10:17:35.127762+0000","last_active":"2024-09-21T10:18:40.166323+0000","last_peered":"2024-09-21T10:18:40.166323+0000","last_clean":"2024-09-21T10:17:34.135019+0000","last_became_active":"2024-09-21T10:17:35.127762+0000","last_became_peered":"2024-09-21T10:17:35.127762+0000","last_unstale":"2024-09-21T10:18:40.166323+0000","last_undegraded":"2024-09-21T10:17:35.126607+0000","last_fullsized":"2024-09-21T10:17:35.126515+0000","mapping_epoch":281,"log_start":"275'10032","ondisk_log_start":"275'10032","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5265,"log_dups_size":0,"ondisk_log_size":5265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:44:22.448438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24313856,"num_objects":371,"num_object_clones":0,"num_object_copies":1113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15297,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":9043968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":371}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":30,"reported_epoch":298,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:37.705186+0000","last_change":"2024-09-21T10:18:32.647924+0000","last_active":"2024-09-21T10:18:37.705186+0000","last_peered":"2024-09-21T10:18:37.705186+0000","last_clean":"2024-09-21T10:18:37.705186+0000","last_became_active":"2024-09-21T10:18:32.647444+0000","last_became_peered":"2024-09-21T10:18:32.647444+0000","last_unstale":"2024-09-21T10:18:37.705186+0000","last_undegraded":"2024-09-21T10:18:37.705186+0000","last_fullsized":"2024-09-21T10:18:37.705186+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:43.786452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,7],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":124,"reported_epoch":298,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:37.685931+0000","last_change":"2024-09-21T10:18:32.650449+0000","last_active":"2024-09-21T10:18:37.685931+0000","last_peered":"2024-09-21T10:18:37.685931+0000","last_clean":"2024-09-21T10:18:37.685931+0000","last_became_active":"2024-09-21T10:18:32.649980+0000","last_became_peered":"2024-09-21T10:18:32.649980+0000","last_unstale":"2024-09-21T10:18:37.685931+0000","last_undegraded":"2024-09-21T10:18:37.685931+0000","last_fullsized":"2024-09-21T10:18:37.685931+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:11.258551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,7],"avail_no_missing":["6","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.a","version":"299'15524","reported_seq":15968,"reported_epoch":299,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:43.567356+0000","last_change":"2024-09-21T10:17:35.130954+0000","last_active":"2024-09-21T10:18:43.567356+0000","last_peered":"2024-09-21T10:18:43.567356+0000","last_clean":"2024-09-21T10:17:34.113182+0000","last_became_active":"2024-09-21T10:17:35.130954+0000","last_became_peered":"2024-09-21T10:17:35.130954+0000","last_unstale":"2024-09-21T10:18:43.567356+0000","last_undegraded":"2024-09-21T10:17:35.121199+0000","last_fullsized":"2024-09-21T10:17:35.121083+0000","mapping_epoch":281,"log_start":"268'8911","ondisk_log_start":"268'8911","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6613,"log_dups_size":0,"ondisk_log_size":6613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:13:03.054152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11665408,"num_objects":178,"num_object_clones":0,"num_object_copies":534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":178,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15524,"num_write_kb":502464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":178}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":196,"reported_epoch":298,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:37.704989+0000","last_change":"2024-09-21T10:18:32.646304+0000","last_active":"2024-09-21T10:18:37.704989+0000","last_peered":"2024-09-21T10:18:37.704989+0000","last_clean":"2024-09-21T10:18:37.704989+0000","last_became_active":"2024-09-21T10:18:32.645618+0000","last_became_peered":"2024-09-21T10:18:32.645618+0000","last_unstale":"2024-09-21T10:18:37.704989+0000","last_undegraded":"2024-09-21T10:18:37.704989+0000","last_fullsized":"2024-09-21T10:18:37.704989+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:34.821649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,7],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":301,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.706145+0000","last_change":"2024-09-21T10:13:09.578819+0000","last_active":"2024-09-21T10:18:37.706145+0000","last_peered":"2024-09-21T10:18:37.706145+0000","last_clean":"2024-09-21T10:18:37.706145+0000","last_became_active":"2024-09-21T10:13:09.576707+0000","last_became_peered":"2024-09-21T10:13:09.576707+0000","last_unstale":"2024-09-21T10:18:37.706145+0000","last_undegraded":"2024-09-21T10:18:37.706145+0000","last_fullsized":"2024-09-21T10:18:37.706145+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:19.175846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"299'15268","reported_seq":15395,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:40.192537+0000","last_change":"2024-09-21T10:14:32.771740+0000","last_active":"2024-09-21T10:18:40.192537+0000","last_peered":"2024-09-21T10:18:40.192537+0000","last_clean":"2024-09-21T10:18:40.192537+0000","last_became_active":"2024-09-21T10:14:32.771298+0000","last_became_peered":"2024-09-21T10:14:32.771298+0000","last_unstale":"2024-09-21T10:18:40.192537+0000","last_undegraded":"2024-09-21T10:18:40.192537+0000","last_fullsized":"2024-09-21T10:18:40.192537+0000","mapping_epoch":248,"log_start":"275'9926","ondisk_log_start":"275'9926","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5342,"log_dups_size":0,"ondisk_log_size":5342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:27.287728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25821184,"num_objects":394,"num_object_clones":0,"num_object_copies":1182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15268,"num_write_kb":501184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":286,"reported_epoch":298,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:37.687154+0000","last_change":"2024-09-21T10:17:35.175102+0000","last_active":"2024-09-21T10:18:37.687154+0000","last_peered":"2024-09-21T10:18:37.687154+0000","last_clean":"2024-09-21T10:17:25.110838+0000","last_became_active":"2024-09-21T10:17:35.175102+0000","last_became_peered":"2024-09-21T10:17:35.175102+0000","last_unstale":"2024-09-21T10:18:37.687154+0000","last_undegraded":"2024-09-21T10:17:35.122397+0000","last_fullsized":"2024-09-21T10:17:35.122269+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:57.830633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":296,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:38.696621+0000","last_change":"2024-09-21T10:18:32.653382+0000","last_active":"2024-09-21T10:18:38.696621+0000","last_peered":"2024-09-21T10:18:38.696621+0000","last_clean":"2024-09-21T10:18:38.696621+0000","last_became_active":"2024-09-21T10:18:32.652907+0000","last_became_peered":"2024-09-21T10:18:32.652907+0000","last_unstale":"2024-09-21T10:18:38.696621+0000","last_undegraded":"2024-09-21T10:18:38.696621+0000","last_fullsized":"2024-09-21T10:18:38.696621+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:51.878587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"299'15421","reported_seq":15540,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:44.114860+0000","last_change":"2024-09-21T10:14:28.438058+0000","last_active":"2024-09-21T10:18:44.114860+0000","last_peered":"2024-09-21T10:18:44.114860+0000","last_clean":"2024-09-21T10:18:44.114860+0000","last_became_active":"2024-09-21T10:14:28.437772+0000","last_became_peered":"2024-09-21T10:14:28.437772+0000","last_unstale":"2024-09-21T10:18:44.114860+0000","last_undegraded":"2024-09-21T10:18:44.114860+0000","last_fullsized":"2024-09-21T10:18:44.114860+0000","mapping_epoch":245,"log_start":"267'8069","ondisk_log_start":"267'8069","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7352,"log_dups_size":0,"ondisk_log_size":7352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:14.437061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7667712,"num_objects":117,"num_object_clones":0,"num_object_copies":351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15421,"num_write_kb":497216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":107,"reported_epoch":299,"state":"active+undersized","last_fresh":"2024-09-21T10:18:38.689280+0000","last_change":"2024-09-21T10:17:35.125321+0000","last_active":"2024-09-21T10:18:38.689280+0000","last_peered":"2024-09-21T10:18:38.689280+0000","last_clean":"2024-09-21T10:03:19.757236+0000","last_became_active":"2024-09-21T10:17:35.125321+0000","last_became_peered":"2024-09-21T10:17:35.125321+0000","last_unstale":"2024-09-21T10:18:38.689280+0000","last_undegraded":"2024-09-21T10:18:38.689280+0000","last_fullsized":"2024-09-21T10:17:35.122818+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:18.261047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":70,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.688087+0000","last_change":"2024-09-21T10:16:35.252681+0000","last_active":"2024-09-21T10:18:38.688087+0000","last_peered":"2024-09-21T10:18:38.688087+0000","last_clean":"2024-09-21T10:18:38.688087+0000","last_became_active":"2024-09-21T10:16:35.252333+0000","last_became_peered":"2024-09-21T10:16:35.252333+0000","last_unstale":"2024-09-21T10:18:38.688087+0000","last_undegraded":"2024-09-21T10:18:38.688087+0000","last_fullsized":"2024-09-21T10:18:38.688087+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:43.211846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":388,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.685868+0000","last_change":"2024-09-21T10:18:34.659194+0000","last_active":"2024-09-21T10:18:37.685868+0000","last_peered":"2024-09-21T10:18:37.685868+0000","last_clean":"2024-09-21T10:18:37.685868+0000","last_became_active":"2024-09-21T10:18:34.658833+0000","last_became_peered":"2024-09-21T10:18:34.658833+0000","last_unstale":"2024-09-21T10:18:37.685868+0000","last_undegraded":"2024-09-21T10:18:37.685868+0000","last_fullsized":"2024-09-21T10:18:37.685868+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:18:21.604320+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":74,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.f","version":"299'15396","reported_seq":15516,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:43.548888+0000","last_change":"2024-09-21T10:14:28.441282+0000","last_active":"2024-09-21T10:18:43.548888+0000","last_peered":"2024-09-21T10:18:43.548888+0000","last_clean":"2024-09-21T10:18:43.548888+0000","last_became_active":"2024-09-21T10:14:28.440767+0000","last_became_peered":"2024-09-21T10:14:28.440767+0000","last_unstale":"2024-09-21T10:18:43.548888+0000","last_undegraded":"2024-09-21T10:18:43.548888+0000","last_fullsized":"2024-09-21T10:18:43.548888+0000","mapping_epoch":245,"log_start":"268'8805","ondisk_log_start":"268'8805","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:56:18.651559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10223616,"num_objects":156,"num_object_clones":0,"num_object_copies":468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15396,"num_write_kb":497664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":490,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.689043+0000","last_change":"2024-09-21T10:13:09.594302+0000","last_active":"2024-09-21T10:18:38.689043+0000","last_peered":"2024-09-21T10:18:38.689043+0000","last_clean":"2024-09-21T10:18:38.689043+0000","last_became_active":"2024-09-21T10:13:09.576948+0000","last_became_peered":"2024-09-21T10:13:09.576948+0000","last_unstale":"2024-09-21T10:18:38.689043+0000","last_undegraded":"2024-09-21T10:18:38.689043+0000","last_fullsized":"2024-09-21T10:18:38.689043+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:10.360060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":267,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.704735+0000","last_change":"2024-09-21T10:03:53.839803+0000","last_active":"2024-09-21T10:18:37.704735+0000","last_peered":"2024-09-21T10:18:37.704735+0000","last_clean":"2024-09-21T10:18:37.704735+0000","last_became_active":"2024-09-21T10:03:53.839449+0000","last_became_peered":"2024-09-21T10:03:53.839449+0000","last_unstale":"2024-09-21T10:18:37.704735+0000","last_undegraded":"2024-09-21T10:18:37.704735+0000","last_fullsized":"2024-09-21T10:18:37.704735+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:41.057164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":392,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.685559+0000","last_change":"2024-09-21T10:18:34.663987+0000","last_active":"2024-09-21T10:18:37.685559+0000","last_peered":"2024-09-21T10:18:37.685559+0000","last_clean":"2024-09-21T10:18:37.685559+0000","last_became_active":"2024-09-21T10:18:34.663451+0000","last_became_peered":"2024-09-21T10:18:34.663451+0000","last_unstale":"2024-09-21T10:18:37.685559+0000","last_undegraded":"2024-09-21T10:18:37.685559+0000","last_fullsized":"2024-09-21T10:18:37.685559+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:27.498931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:27.498931+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.e","version":"299'15535","reported_seq":16004,"reported_epoch":299,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:44.141674+0000","last_change":"2024-09-21T10:17:35.130144+0000","last_active":"2024-09-21T10:18:44.141674+0000","last_peered":"2024-09-21T10:18:44.141674+0000","last_clean":"2024-09-21T10:17:34.129564+0000","last_became_active":"2024-09-21T10:17:35.130144+0000","last_became_peered":"2024-09-21T10:17:35.130144+0000","last_unstale":"2024-09-21T10:18:44.141674+0000","last_undegraded":"2024-09-21T10:17:35.124645+0000","last_fullsized":"2024-09-21T10:17:35.124526+0000","mapping_epoch":281,"log_start":"267'8209","ondisk_log_start":"267'8209","created":245,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7326,"log_dups_size":0,"ondisk_log_size":7326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:32.382462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8978432,"num_objects":137,"num_object_clones":0,"num_object_copies":411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":137,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15535,"num_write_kb":501504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":166,"num_bytes_recovered":10878976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":137}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":287,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.696799+0000","last_change":"2024-09-21T10:13:09.578151+0000","last_active":"2024-09-21T10:18:38.696799+0000","last_peered":"2024-09-21T10:18:38.696799+0000","last_clean":"2024-09-21T10:18:38.696799+0000","last_became_active":"2024-09-21T10:13:09.577233+0000","last_became_peered":"2024-09-21T10:13:09.577233+0000","last_unstale":"2024-09-21T10:18:38.696799+0000","last_undegraded":"2024-09-21T10:18:38.696799+0000","last_fullsized":"2024-09-21T10:18:38.696799+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:00.627590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":62,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.705175+0000","last_change":"2024-09-21T10:14:32.771000+0000","last_active":"2024-09-21T10:18:37.705175+0000","last_peered":"2024-09-21T10:18:37.705175+0000","last_clean":"2024-09-21T10:18:37.705175+0000","last_became_active":"2024-09-21T10:14:32.770562+0000","last_became_peered":"2024-09-21T10:14:32.770562+0000","last_unstale":"2024-09-21T10:18:37.705175+0000","last_undegraded":"2024-09-21T10:18:37.705175+0000","last_fullsized":"2024-09-21T10:18:37.705175+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:47.956518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"299'15170","reported_seq":22319,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:39.807573+0000","last_change":"2024-09-21T10:17:49.596487+0000","last_active":"2024-09-21T10:18:39.807573+0000","last_peered":"2024-09-21T10:18:39.807573+0000","last_clean":"2024-09-21T10:18:39.807573+0000","last_became_active":"2024-09-21T10:17:49.571549+0000","last_became_peered":"2024-09-21T10:17:49.571549+0000","last_unstale":"2024-09-21T10:18:39.807573+0000","last_undegraded":"2024-09-21T10:18:39.807573+0000","last_fullsized":"2024-09-21T10:18:39.807573+0000","mapping_epoch":286,"log_start":"268'8581","ondisk_log_start":"268'8581","created":245,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6589,"log_dups_size":0,"ondisk_log_size":6589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:07.398175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24707096,"num_objects":378,"num_object_clones":0,"num_object_copies":1134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":378,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15170,"num_write_kb":497473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3528,"num_bytes_recovered":231145496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":269,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.706324+0000","last_change":"2024-09-21T10:03:19.767897+0000","last_active":"2024-09-21T10:18:37.706324+0000","last_peered":"2024-09-21T10:18:37.706324+0000","last_clean":"2024-09-21T10:18:37.706324+0000","last_became_active":"2024-09-21T10:03:19.767541+0000","last_became_peered":"2024-09-21T10:03:19.767541+0000","last_unstale":"2024-09-21T10:18:37.706324+0000","last_undegraded":"2024-09-21T10:18:37.706324+0000","last_fullsized":"2024-09-21T10:18:37.706324+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:04.738237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":285,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.705042+0000","last_change":"2024-09-21T10:13:09.579444+0000","last_active":"2024-09-21T10:18:37.705042+0000","last_peered":"2024-09-21T10:18:37.705042+0000","last_clean":"2024-09-21T10:18:37.705042+0000","last_became_active":"2024-09-21T10:13:09.579162+0000","last_became_peered":"2024-09-21T10:13:09.579162+0000","last_unstale":"2024-09-21T10:18:37.705042+0000","last_undegraded":"2024-09-21T10:18:37.705042+0000","last_fullsized":"2024-09-21T10:18:37.705042+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.051244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":388,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.685969+0000","last_change":"2024-09-21T10:18:34.663241+0000","last_active":"2024-09-21T10:18:37.685969+0000","last_peered":"2024-09-21T10:18:37.685969+0000","last_clean":"2024-09-21T10:18:37.685969+0000","last_became_active":"2024-09-21T10:18:34.662897+0000","last_became_peered":"2024-09-21T10:18:34.662897+0000","last_unstale":"2024-09-21T10:18:37.685969+0000","last_undegraded":"2024-09-21T10:18:37.685969+0000","last_fullsized":"2024-09-21T10:18:37.685969+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:24.636770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:24.636770+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":74,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"299'15022","reported_seq":15086,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:39.839279+0000","last_change":"2024-09-21T10:14:28.439503+0000","last_active":"2024-09-21T10:18:39.839279+0000","last_peered":"2024-09-21T10:18:39.839279+0000","last_clean":"2024-09-21T10:18:39.839279+0000","last_became_active":"2024-09-21T10:14:28.438413+0000","last_became_peered":"2024-09-21T10:14:28.438413+0000","last_unstale":"2024-09-21T10:18:39.839279+0000","last_undegraded":"2024-09-21T10:18:39.839279+0000","last_fullsized":"2024-09-21T10:18:39.839279+0000","mapping_epoch":245,"log_start":"268'8474","ondisk_log_start":"268'8474","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:24.386364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27131904,"num_objects":414,"num_object_clones":0,"num_object_copies":1242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15022,"num_write_kb":493952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":481,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.688821+0000","last_change":"2024-09-21T10:18:31.643054+0000","last_active":"2024-09-21T10:18:38.688821+0000","last_peered":"2024-09-21T10:18:38.688821+0000","last_clean":"2024-09-21T10:18:38.688821+0000","last_became_active":"2024-09-21T10:18:31.642707+0000","last_became_peered":"2024-09-21T10:18:31.642707+0000","last_unstale":"2024-09-21T10:18:38.688821+0000","last_undegraded":"2024-09-21T10:18:38.688821+0000","last_fullsized":"2024-09-21T10:18:38.688821+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:31.806320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":180,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.757420+0000","last_change":"2024-09-21T10:13:09.586643+0000","last_active":"2024-09-21T10:18:38.757420+0000","last_peered":"2024-09-21T10:18:38.757420+0000","last_clean":"2024-09-21T10:18:38.757420+0000","last_became_active":"2024-09-21T10:13:09.586283+0000","last_became_peered":"2024-09-21T10:13:09.586283+0000","last_unstale":"2024-09-21T10:18:38.757420+0000","last_undegraded":"2024-09-21T10:18:38.757420+0000","last_fullsized":"2024-09-21T10:18:38.757420+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:20.002373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"299'15414","reported_seq":24400,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:41.463647+0000","last_change":"2024-09-21T10:18:32.707153+0000","last_active":"2024-09-21T10:18:41.463647+0000","last_peered":"2024-09-21T10:18:41.463647+0000","last_clean":"2024-09-21T10:18:41.463647+0000","last_became_active":"2024-09-21T10:18:32.646421+0000","last_became_peered":"2024-09-21T10:18:32.646421+0000","last_unstale":"2024-09-21T10:18:41.463647+0000","last_undegraded":"2024-09-21T10:18:41.463647+0000","last_fullsized":"2024-09-21T10:18:41.463647+0000","mapping_epoch":294,"log_start":"264'5400","ondisk_log_start":"264'5400","created":245,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":10014,"log_dups_size":0,"ondisk_log_size":10014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:46.429182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20709376,"num_objects":316,"num_object_clones":0,"num_object_copies":948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":316,"num_objects_unfound":0,"num_objects_dirty":316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15414,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4424,"num_bytes_recovered":289800192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":316}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":388,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.686294+0000","last_change":"2024-09-21T10:18:34.663785+0000","last_active":"2024-09-21T10:18:37.686294+0000","last_peered":"2024-09-21T10:18:37.686294+0000","last_clean":"2024-09-21T10:18:37.686294+0000","last_became_active":"2024-09-21T10:18:34.662991+0000","last_became_peered":"2024-09-21T10:18:34.662991+0000","last_unstale":"2024-09-21T10:18:37.686294+0000","last_undegraded":"2024-09-21T10:18:37.686294+0000","last_fullsized":"2024-09-21T10:18:37.686294+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:14.613041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:14.613041+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":74,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":446,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:38.691831+0000","last_change":"2024-09-21T10:18:34.667971+0000","last_active":"2024-09-21T10:18:38.691831+0000","last_peered":"2024-09-21T10:18:38.691831+0000","last_clean":"2024-09-21T10:18:38.691831+0000","last_became_active":"2024-09-21T10:18:34.661067+0000","last_became_peered":"2024-09-21T10:18:34.661067+0000","last_unstale":"2024-09-21T10:18:38.691831+0000","last_undegraded":"2024-09-21T10:18:38.691831+0000","last_fullsized":"2024-09-21T10:18:38.691831+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:19:11.721310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":426,"reported_epoch":299,"state":"active+undersized","last_fresh":"2024-09-21T10:18:38.689019+0000","last_change":"2024-09-21T10:17:35.126459+0000","last_active":"2024-09-21T10:18:38.689019+0000","last_peered":"2024-09-21T10:18:38.689019+0000","last_clean":"2024-09-21T10:17:25.100810+0000","last_became_active":"2024-09-21T10:17:35.126459+0000","last_became_peered":"2024-09-21T10:17:35.126459+0000","last_unstale":"2024-09-21T10:18:38.689019+0000","last_undegraded":"2024-09-21T10:18:38.689019+0000","last_fullsized":"2024-09-21T10:17:35.124041+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:02.350739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":253,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.686563+0000","last_change":"2024-09-21T09:50:22.931349+0000","last_active":"2024-09-21T10:18:37.686563+0000","last_peered":"2024-09-21T10:18:37.686563+0000","last_clean":"2024-09-21T10:18:37.686563+0000","last_became_active":"2024-09-21T09:50:22.931038+0000","last_became_peered":"2024-09-21T09:50:22.931038+0000","last_unstale":"2024-09-21T10:18:37.686563+0000","last_undegraded":"2024-09-21T10:18:37.686563+0000","last_fullsized":"2024-09-21T10:18:37.686563+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:11.300887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":388,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.685962+0000","last_change":"2024-09-21T10:18:34.659585+0000","last_active":"2024-09-21T10:18:37.685962+0000","last_peered":"2024-09-21T10:18:37.685962+0000","last_clean":"2024-09-21T10:18:37.685962+0000","last_became_active":"2024-09-21T10:18:34.659230+0000","last_became_peered":"2024-09-21T10:18:34.659230+0000","last_unstale":"2024-09-21T10:18:37.685962+0000","last_undegraded":"2024-09-21T10:18:37.685962+0000","last_fullsized":"2024-09-21T10:18:37.685962+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:20.629033+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:20.629033+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"299'15371","reported_seq":16698,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:44.092495+0000","last_change":"2024-09-21T10:18:38.833231+0000","last_active":"2024-09-21T10:18:44.092495+0000","last_peered":"2024-09-21T10:18:44.092495+0000","last_clean":"2024-09-21T10:18:44.092495+0000","last_became_active":"2024-09-21T10:18:38.764257+0000","last_became_peered":"2024-09-21T10:18:38.764257+0000","last_unstale":"2024-09-21T10:18:44.092495+0000","last_undegraded":"2024-09-21T10:18:44.092495+0000","last_fullsized":"2024-09-21T10:18:44.092495+0000","mapping_epoch":298,"log_start":"268'8584","ondisk_log_start":"268'8584","created":245,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6787,"log_dups_size":0,"ondisk_log_size":6787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:01.425679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8716288,"num_objects":133,"num_object_clones":0,"num_object_copies":399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15371,"num_write_kb":496128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":620,"num_bytes_recovered":40370176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":936,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.685519+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:18:37.685519+0000","last_peered":"2024-09-21T10:18:37.685519+0000","last_clean":"2024-09-21T10:18:37.685519+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:18:37.685519+0000","last_undegraded":"2024-09-21T10:18:37.685519+0000","last_fullsized":"2024-09-21T10:18:37.685519+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":71,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.687862+0000","last_change":"2024-09-21T10:16:35.252630+0000","last_active":"2024-09-21T10:18:38.687862+0000","last_peered":"2024-09-21T10:18:38.687862+0000","last_clean":"2024-09-21T10:18:38.687862+0000","last_became_active":"2024-09-21T10:16:35.252279+0000","last_became_peered":"2024-09-21T10:16:35.252279+0000","last_unstale":"2024-09-21T10:18:38.687862+0000","last_undegraded":"2024-09-21T10:18:38.687862+0000","last_fullsized":"2024-09-21T10:18:38.687862+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:12.918989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":186,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:38.757290+0000","last_change":"2024-09-21T10:18:32.648061+0000","last_active":"2024-09-21T10:18:38.757290+0000","last_peered":"2024-09-21T10:18:38.757290+0000","last_clean":"2024-09-21T10:18:38.757290+0000","last_became_active":"2024-09-21T10:18:32.646282+0000","last_became_peered":"2024-09-21T10:18:32.646282+0000","last_unstale":"2024-09-21T10:18:38.757290+0000","last_undegraded":"2024-09-21T10:18:38.757290+0000","last_fullsized":"2024-09-21T10:18:38.757290+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:29.602760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":388,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.686076+0000","last_change":"2024-09-21T10:18:34.662819+0000","last_active":"2024-09-21T10:18:37.686076+0000","last_peered":"2024-09-21T10:18:37.686076+0000","last_clean":"2024-09-21T10:18:37.686076+0000","last_became_active":"2024-09-21T10:18:34.662445+0000","last_became_peered":"2024-09-21T10:18:34.662445+0000","last_unstale":"2024-09-21T10:18:37.686076+0000","last_undegraded":"2024-09-21T10:18:37.686076+0000","last_fullsized":"2024-09-21T10:18:37.686076+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:23.626014+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:23.626014+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.4","version":"299'15081","reported_seq":15143,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:39.782329+0000","last_change":"2024-09-21T10:14:32.642821+0000","last_active":"2024-09-21T10:18:39.782329+0000","last_peered":"2024-09-21T10:18:39.782329+0000","last_clean":"2024-09-21T10:18:39.782329+0000","last_became_active":"2024-09-21T10:14:32.642404+0000","last_became_peered":"2024-09-21T10:14:32.642404+0000","last_unstale":"2024-09-21T10:18:39.782329+0000","last_undegraded":"2024-09-21T10:18:39.782329+0000","last_fullsized":"2024-09-21T10:18:39.782329+0000","mapping_epoch":248,"log_start":"266'7529","ondisk_log_start":"266'7529","created":245,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7552,"log_dups_size":0,"ondisk_log_size":7552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:02.051331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15081,"num_write_kb":494784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1022,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.691569+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:18:38.691569+0000","last_peered":"2024-09-21T10:18:38.691569+0000","last_clean":"2024-09-21T10:18:38.691569+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:18:38.691569+0000","last_undegraded":"2024-09-21T10:18:38.691569+0000","last_fullsized":"2024-09-21T10:18:38.691569+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":180,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.756826+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:18:38.756826+0000","last_peered":"2024-09-21T10:18:38.756826+0000","last_clean":"2024-09-21T10:18:38.756826+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:18:38.756826+0000","last_undegraded":"2024-09-21T10:18:38.756826+0000","last_fullsized":"2024-09-21T10:18:38.756826+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":190,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:38.757433+0000","last_change":"2024-09-21T10:18:32.647639+0000","last_active":"2024-09-21T10:18:38.757433+0000","last_peered":"2024-09-21T10:18:38.757433+0000","last_clean":"2024-09-21T10:18:38.757433+0000","last_became_active":"2024-09-21T10:18:32.647214+0000","last_became_peered":"2024-09-21T10:18:32.647214+0000","last_unstale":"2024-09-21T10:18:38.757433+0000","last_undegraded":"2024-09-21T10:18:38.757433+0000","last_fullsized":"2024-09-21T10:18:38.757433+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:24.352069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,7],"avail_no_missing":["1","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":388,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.685754+0000","last_change":"2024-09-21T10:18:34.659201+0000","last_active":"2024-09-21T10:18:37.685754+0000","last_peered":"2024-09-21T10:18:37.685754+0000","last_clean":"2024-09-21T10:18:37.685754+0000","last_became_active":"2024-09-21T10:18:34.658854+0000","last_became_peered":"2024-09-21T10:18:34.658854+0000","last_unstale":"2024-09-21T10:18:37.685754+0000","last_undegraded":"2024-09-21T10:18:37.685754+0000","last_fullsized":"2024-09-21T10:18:37.685754+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:16.616226+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:16.616226+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.7","version":"299'15135","reported_seq":26406,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:39.829033+0000","last_change":"2024-09-21T10:17:12.636504+0000","last_active":"2024-09-21T10:18:39.829033+0000","last_peered":"2024-09-21T10:18:39.829033+0000","last_clean":"2024-09-21T10:18:39.829033+0000","last_became_active":"2024-09-21T10:17:12.615672+0000","last_became_peered":"2024-09-21T10:17:12.615672+0000","last_unstale":"2024-09-21T10:18:39.829033+0000","last_undegraded":"2024-09-21T10:18:39.829033+0000","last_fullsized":"2024-09-21T10:18:39.829033+0000","mapping_epoch":277,"log_start":"268'8585","ondisk_log_start":"268'8585","created":245,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6550,"log_dups_size":0,"ondisk_log_size":6550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:54.239276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15135,"num_write_kb":496512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5594,"num_bytes_recovered":366608384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1010,"reported_epoch":299,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:38.756666+0000","last_change":"2024-09-21T10:17:35.127768+0000","last_active":"2024-09-21T10:18:38.756666+0000","last_peered":"2024-09-21T10:18:38.756666+0000","last_clean":"2024-09-21T10:12:59.503459+0000","last_became_active":"2024-09-21T10:17:35.127768+0000","last_became_peered":"2024-09-21T10:17:35.127768+0000","last_unstale":"2024-09-21T10:18:38.756666+0000","last_undegraded":"2024-09-21T10:17:35.126492+0000","last_fullsized":"2024-09-21T10:17:35.126379+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:37:30.134026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":213,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.691661+0000","last_change":"2024-09-21T10:16:35.253445+0000","last_active":"2024-09-21T10:18:38.691661+0000","last_peered":"2024-09-21T10:18:38.691661+0000","last_clean":"2024-09-21T10:18:38.691661+0000","last_became_active":"2024-09-21T10:16:35.253121+0000","last_became_peered":"2024-09-21T10:16:35.253121+0000","last_unstale":"2024-09-21T10:18:38.691661+0000","last_undegraded":"2024-09-21T10:18:38.691661+0000","last_fullsized":"2024-09-21T10:18:38.691661+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:36.816904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":180,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.757036+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:18:38.757036+0000","last_peered":"2024-09-21T10:18:38.757036+0000","last_clean":"2024-09-21T10:18:38.757036+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:18:38.757036+0000","last_undegraded":"2024-09-21T10:18:38.757036+0000","last_fullsized":"2024-09-21T10:18:38.757036+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":388,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.685982+0000","last_change":"2024-09-21T10:18:34.659260+0000","last_active":"2024-09-21T10:18:37.685982+0000","last_peered":"2024-09-21T10:18:37.685982+0000","last_clean":"2024-09-21T10:18:37.685982+0000","last_became_active":"2024-09-21T10:18:34.658851+0000","last_became_peered":"2024-09-21T10:18:34.658851+0000","last_unstale":"2024-09-21T10:18:37.685982+0000","last_undegraded":"2024-09-21T10:18:37.685982+0000","last_fullsized":"2024-09-21T10:18:37.685982+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:19.586604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:19.586604+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.1","version":"299'15024","reported_seq":15088,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:39.792125+0000","last_change":"2024-09-21T10:14:28.437169+0000","last_active":"2024-09-21T10:18:39.792125+0000","last_peered":"2024-09-21T10:18:39.792125+0000","last_clean":"2024-09-21T10:18:39.792125+0000","last_became_active":"2024-09-21T10:14:28.436873+0000","last_became_peered":"2024-09-21T10:14:28.436873+0000","last_unstale":"2024-09-21T10:18:39.792125+0000","last_undegraded":"2024-09-21T10:18:39.792125+0000","last_fullsized":"2024-09-21T10:18:39.792125+0000","mapping_epoch":245,"log_start":"266'7457","ondisk_log_start":"266'7457","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":7567,"log_dups_size":0,"ondisk_log_size":7567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:16.516429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27787264,"num_objects":424,"num_object_clones":0,"num_object_copies":1272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15024,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":123,"reported_epoch":299,"state":"active+undersized","last_fresh":"2024-09-21T10:18:38.689428+0000","last_change":"2024-09-21T10:17:35.132728+0000","last_active":"2024-09-21T10:18:38.689428+0000","last_peered":"2024-09-21T10:18:38.689428+0000","last_clean":"2024-09-21T10:03:19.766344+0000","last_became_active":"2024-09-21T10:17:35.132728+0000","last_became_peered":"2024-09-21T10:17:35.132728+0000","last_unstale":"2024-09-21T10:18:38.689428+0000","last_undegraded":"2024-09-21T10:18:38.689428+0000","last_fullsized":"2024-09-21T10:17:35.122807+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:44:50.875126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":789,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.757330+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:18:38.757330+0000","last_peered":"2024-09-21T10:18:38.757330+0000","last_clean":"2024-09-21T10:18:38.757330+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:18:38.757330+0000","last_undegraded":"2024-09-21T10:18:38.757330+0000","last_fullsized":"2024-09-21T10:18:38.757330+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":198,"reported_epoch":299,"state":"active+undersized","last_fresh":"2024-09-21T10:18:38.756888+0000","last_change":"2024-09-21T10:17:35.127413+0000","last_active":"2024-09-21T10:18:38.756888+0000","last_peered":"2024-09-21T10:18:38.756888+0000","last_clean":"2024-09-21T10:17:25.127289+0000","last_became_active":"2024-09-21T10:17:35.127413+0000","last_became_peered":"2024-09-21T10:17:35.127413+0000","last_unstale":"2024-09-21T10:18:38.756888+0000","last_undegraded":"2024-09-21T10:18:38.756888+0000","last_fullsized":"2024-09-21T10:17:35.126028+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:57.483438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":388,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.686363+0000","last_change":"2024-09-21T10:18:34.669804+0000","last_active":"2024-09-21T10:18:37.686363+0000","last_peered":"2024-09-21T10:18:37.686363+0000","last_clean":"2024-09-21T10:18:37.686363+0000","last_became_active":"2024-09-21T10:18:34.669541+0000","last_became_peered":"2024-09-21T10:18:34.669541+0000","last_unstale":"2024-09-21T10:18:37.686363+0000","last_undegraded":"2024-09-21T10:18:37.686363+0000","last_fullsized":"2024-09-21T10:18:37.686363+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:18.617770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:18.617770+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.0","version":"299'15380","reported_seq":15772,"reported_epoch":299,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:40.205613+0000","last_change":"2024-09-21T10:17:35.127386+0000","last_active":"2024-09-21T10:18:40.205613+0000","last_peered":"2024-09-21T10:18:40.205613+0000","last_clean":"2024-09-21T10:17:34.093837+0000","last_became_active":"2024-09-21T10:17:35.127386+0000","last_became_peered":"2024-09-21T10:17:35.127386+0000","last_unstale":"2024-09-21T10:18:40.205613+0000","last_undegraded":"2024-09-21T10:17:35.125205+0000","last_fullsized":"2024-09-21T10:17:35.125090+0000","mapping_epoch":281,"log_start":"275'10063","ondisk_log_start":"275'10063","created":245,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":5317,"log_dups_size":0,"ondisk_log_size":5317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:42.679516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22282240,"num_objects":340,"num_object_clones":0,"num_object_copies":1020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":340,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15380,"num_write_kb":503040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":340}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":191,"reported_epoch":299,"state":"active+undersized","last_fresh":"2024-09-21T10:18:38.757205+0000","last_change":"2024-09-21T10:17:35.130553+0000","last_active":"2024-09-21T10:18:38.757205+0000","last_peered":"2024-09-21T10:18:38.757205+0000","last_clean":"2024-09-21T10:17:25.110962+0000","last_became_active":"2024-09-21T10:17:35.130553+0000","last_became_peered":"2024-09-21T10:17:35.130553+0000","last_unstale":"2024-09-21T10:18:38.757205+0000","last_undegraded":"2024-09-21T10:18:38.757205+0000","last_fullsized":"2024-09-21T10:17:35.127855+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:58.373232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":782,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.687258+0000","last_change":"2024-09-21T10:13:09.604050+0000","last_active":"2024-09-21T10:18:37.687258+0000","last_peered":"2024-09-21T10:18:37.687258+0000","last_clean":"2024-09-21T10:18:37.687258+0000","last_became_active":"2024-09-21T10:13:09.583593+0000","last_became_peered":"2024-09-21T10:13:09.583593+0000","last_unstale":"2024-09-21T10:18:37.687258+0000","last_undegraded":"2024-09-21T10:18:37.687258+0000","last_fullsized":"2024-09-21T10:18:37.687258+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:35.420105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":75,"reported_epoch":299,"state":"active+undersized","last_fresh":"2024-09-21T10:18:38.687973+0000","last_change":"2024-09-21T10:17:35.176265+0000","last_active":"2024-09-21T10:18:38.687973+0000","last_peered":"2024-09-21T10:18:38.687973+0000","last_clean":"2024-09-21T10:17:25.114921+0000","last_became_active":"2024-09-21T10:17:35.176265+0000","last_became_peered":"2024-09-21T10:17:35.176265+0000","last_unstale":"2024-09-21T10:18:38.687973+0000","last_undegraded":"2024-09-21T10:18:38.687973+0000","last_fullsized":"2024-09-21T10:17:35.128768+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:53.947774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":388,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.686652+0000","last_change":"2024-09-21T10:18:34.669018+0000","last_active":"2024-09-21T10:18:37.686652+0000","last_peered":"2024-09-21T10:18:37.686652+0000","last_clean":"2024-09-21T10:18:37.686652+0000","last_became_active":"2024-09-21T10:18:34.668655+0000","last_became_peered":"2024-09-21T10:18:34.668655+0000","last_unstale":"2024-09-21T10:18:37.686652+0000","last_undegraded":"2024-09-21T10:18:37.686652+0000","last_fullsized":"2024-09-21T10:18:37.686652+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:29.465424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:29.465424+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.3","version":"299'15543","reported_seq":15663,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:43.568446+0000","last_change":"2024-09-21T10:14:28.438913+0000","last_active":"2024-09-21T10:18:43.568446+0000","last_peered":"2024-09-21T10:18:43.568446+0000","last_clean":"2024-09-21T10:18:43.568446+0000","last_became_active":"2024-09-21T10:14:28.438598+0000","last_became_peered":"2024-09-21T10:14:28.438598+0000","last_unstale":"2024-09-21T10:18:43.568446+0000","last_undegraded":"2024-09-21T10:18:43.568446+0000","last_fullsized":"2024-09-21T10:18:43.568446+0000","mapping_epoch":245,"log_start":"268'8931","ondisk_log_start":"268'8931","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6612,"log_dups_size":0,"ondisk_log_size":6612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:29:24.765631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10551296,"num_objects":161,"num_object_clones":0,"num_object_copies":483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15543,"num_write_kb":502528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":485,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:38.689799+0000","last_change":"2024-09-21T10:18:32.649794+0000","last_active":"2024-09-21T10:18:38.689799+0000","last_peered":"2024-09-21T10:18:38.689799+0000","last_clean":"2024-09-21T10:18:38.689799+0000","last_became_active":"2024-09-21T10:18:32.649017+0000","last_became_peered":"2024-09-21T10:18:32.649017+0000","last_unstale":"2024-09-21T10:18:38.689799+0000","last_undegraded":"2024-09-21T10:18:38.689799+0000","last_fullsized":"2024-09-21T10:18:38.689799+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:01:28.600510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,7],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":784,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.757456+0000","last_change":"2024-09-21T10:13:12.569309+0000","last_active":"2024-09-21T10:18:38.757456+0000","last_peered":"2024-09-21T10:18:38.757456+0000","last_clean":"2024-09-21T10:18:38.757456+0000","last_became_active":"2024-09-21T10:13:12.568325+0000","last_became_peered":"2024-09-21T10:13:12.568325+0000","last_unstale":"2024-09-21T10:18:38.757456+0000","last_undegraded":"2024-09-21T10:18:38.757456+0000","last_fullsized":"2024-09-21T10:18:38.757456+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:18.314566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":482,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:38.688649+0000","last_change":"2024-09-21T10:18:32.647878+0000","last_active":"2024-09-21T10:18:38.688649+0000","last_peered":"2024-09-21T10:18:38.688649+0000","last_clean":"2024-09-21T10:18:38.688649+0000","last_became_active":"2024-09-21T10:18:32.646851+0000","last_became_peered":"2024-09-21T10:18:32.646851+0000","last_unstale":"2024-09-21T10:18:38.688649+0000","last_undegraded":"2024-09-21T10:18:38.688649+0000","last_fullsized":"2024-09-21T10:18:38.688649+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:53.462192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,7],"avail_no_missing":["3","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":392,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.685989+0000","last_change":"2024-09-21T10:18:34.662539+0000","last_active":"2024-09-21T10:18:37.685989+0000","last_peered":"2024-09-21T10:18:37.685989+0000","last_clean":"2024-09-21T10:18:37.685989+0000","last_became_active":"2024-09-21T10:18:34.662000+0000","last_became_peered":"2024-09-21T10:18:34.662000+0000","last_unstale":"2024-09-21T10:18:37.685989+0000","last_undegraded":"2024-09-21T10:18:37.685989+0000","last_fullsized":"2024-09-21T10:18:37.685989+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:18:22.603295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:18:22.603295+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2","version":"299'15292","reported_seq":15409,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:43.587225+0000","last_change":"2024-09-21T10:14:28.439188+0000","last_active":"2024-09-21T10:18:43.587225+0000","last_peered":"2024-09-21T10:18:43.587225+0000","last_clean":"2024-09-21T10:18:43.587225+0000","last_became_active":"2024-09-21T10:14:28.438763+0000","last_became_peered":"2024-09-21T10:14:28.438763+0000","last_unstale":"2024-09-21T10:18:43.587225+0000","last_undegraded":"2024-09-21T10:18:43.587225+0000","last_fullsized":"2024-09-21T10:18:43.587225+0000","mapping_epoch":245,"log_start":"268'8727","ondisk_log_start":"268'8727","created":245,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:14:27.417833+0000","last_clean_scrub_stamp":"2024-09-21T10:14:27.417833+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:31.383586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11403264,"num_objects":174,"num_object_clones":0,"num_object_copies":522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15292,"num_write_kb":494912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":190,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:38.757205+0000","last_change":"2024-09-21T10:18:32.646783+0000","last_active":"2024-09-21T10:18:38.757205+0000","last_peered":"2024-09-21T10:18:38.757205+0000","last_clean":"2024-09-21T10:18:38.757205+0000","last_became_active":"2024-09-21T10:18:32.645368+0000","last_became_peered":"2024-09-21T10:18:32.645368+0000","last_unstale":"2024-09-21T10:18:38.757205+0000","last_undegraded":"2024-09-21T10:18:38.757205+0000","last_fullsized":"2024-09-21T10:18:38.757205+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:23.319596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":575,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.686445+0000","last_change":"2024-09-21T10:18:31.655281+0000","last_active":"2024-09-21T10:18:37.686445+0000","last_peered":"2024-09-21T10:18:37.686445+0000","last_clean":"2024-09-21T10:18:37.686445+0000","last_became_active":"2024-09-21T10:18:31.643990+0000","last_became_peered":"2024-09-21T10:18:31.643990+0000","last_unstale":"2024-09-21T10:18:37.686445+0000","last_undegraded":"2024-09-21T10:18:37.686445+0000","last_fullsized":"2024-09-21T10:18:37.686445+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:05.692739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":259,"reported_epoch":298,"state":"active+undersized","last_fresh":"2024-09-21T10:18:37.706199+0000","last_change":"2024-09-21T10:17:35.124983+0000","last_active":"2024-09-21T10:18:37.706199+0000","last_peered":"2024-09-21T10:18:37.706199+0000","last_clean":"2024-09-21T10:17:25.103643+0000","last_became_active":"2024-09-21T10:17:35.124983+0000","last_became_peered":"2024-09-21T10:17:35.124983+0000","last_unstale":"2024-09-21T10:18:37.706199+0000","last_undegraded":"2024-09-21T10:18:37.706199+0000","last_fullsized":"2024-09-21T10:17:35.123358+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:42:17.720690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":294,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.706536+0000","last_change":"2024-09-21T10:15:21.038463+0000","last_active":"2024-09-21T10:18:37.706536+0000","last_peered":"2024-09-21T10:18:37.706536+0000","last_clean":"2024-09-21T10:18:37.706536+0000","last_became_active":"2024-09-21T10:15:21.038061+0000","last_became_peered":"2024-09-21T10:15:21.038061+0000","last_unstale":"2024-09-21T10:18:37.706536+0000","last_undegraded":"2024-09-21T10:18:37.706536+0000","last_fullsized":"2024-09-21T10:18:37.706536+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.241295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":276,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.705286+0000","last_change":"2024-09-21T09:50:22.402298+0000","last_active":"2024-09-21T10:18:37.705286+0000","last_peered":"2024-09-21T10:18:37.705286+0000","last_clean":"2024-09-21T10:18:37.705286+0000","last_became_active":"2024-09-21T09:50:22.401934+0000","last_became_peered":"2024-09-21T09:50:22.401934+0000","last_unstale":"2024-09-21T10:18:37.705286+0000","last_undegraded":"2024-09-21T10:18:37.705286+0000","last_fullsized":"2024-09-21T10:18:37.705286+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:03.273780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":109,"reported_epoch":298,"state":"active+undersized","last_fresh":"2024-09-21T10:18:37.686210+0000","last_change":"2024-09-21T10:17:35.128147+0000","last_active":"2024-09-21T10:18:37.686210+0000","last_peered":"2024-09-21T10:18:37.686210+0000","last_clean":"2024-09-21T10:15:13.965890+0000","last_became_active":"2024-09-21T10:17:35.128147+0000","last_became_peered":"2024-09-21T10:17:35.128147+0000","last_unstale":"2024-09-21T10:18:37.686210+0000","last_undegraded":"2024-09-21T10:18:37.686210+0000","last_fullsized":"2024-09-21T10:17:35.121096+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:48.657757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":41,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.689364+0000","last_change":"2024-09-21T10:18:31.645075+0000","last_active":"2024-09-21T10:18:38.689364+0000","last_peered":"2024-09-21T10:18:38.689364+0000","last_clean":"2024-09-21T10:18:38.689364+0000","last_became_active":"2024-09-21T10:18:31.636967+0000","last_became_peered":"2024-09-21T10:18:31.636967+0000","last_unstale":"2024-09-21T10:18:38.689364+0000","last_undegraded":"2024-09-21T10:18:38.689364+0000","last_fullsized":"2024-09-21T10:18:38.689364+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:34.865028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":185,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:38.756994+0000","last_change":"2024-09-21T10:18:32.647551+0000","last_active":"2024-09-21T10:18:38.756994+0000","last_peered":"2024-09-21T10:18:38.756994+0000","last_clean":"2024-09-21T10:18:38.756994+0000","last_became_active":"2024-09-21T10:18:32.645744+0000","last_became_peered":"2024-09-21T10:18:32.645744+0000","last_unstale":"2024-09-21T10:18:38.756994+0000","last_undegraded":"2024-09-21T10:18:38.756994+0000","last_fullsized":"2024-09-21T10:18:38.756994+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:40.208675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,7],"avail_no_missing":["1","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":413,"reported_epoch":299,"state":"active+clean+remapped","last_fresh":"2024-09-21T10:18:38.688620+0000","last_change":"2024-09-21T10:18:32.649872+0000","last_active":"2024-09-21T10:18:38.688620+0000","last_peered":"2024-09-21T10:18:38.688620+0000","last_clean":"2024-09-21T10:18:38.688620+0000","last_became_active":"2024-09-21T10:18:32.649447+0000","last_became_peered":"2024-09-21T10:18:32.649447+0000","last_unstale":"2024-09-21T10:18:38.688620+0000","last_undegraded":"2024-09-21T10:18:38.688620+0000","last_fullsized":"2024-09-21T10:18:38.688620+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:27.832773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,7],"avail_no_missing":["3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":74,"reported_epoch":299,"state":"active+undersized","last_fresh":"2024-09-21T10:18:38.687940+0000","last_change":"2024-09-21T10:17:35.131689+0000","last_active":"2024-09-21T10:18:38.687940+0000","last_peered":"2024-09-21T10:18:38.687940+0000","last_clean":"2024-09-21T10:17:25.115497+0000","last_became_active":"2024-09-21T10:17:35.131689+0000","last_became_peered":"2024-09-21T10:17:35.131689+0000","last_unstale":"2024-09-21T10:18:38.687940+0000","last_undegraded":"2024-09-21T10:18:38.687940+0000","last_fullsized":"2024-09-21T10:17:35.128889+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:53.272548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":486,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.688620+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:18:38.688620+0000","last_peered":"2024-09-21T10:18:38.688620+0000","last_clean":"2024-09-21T10:18:38.688620+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:18:38.688620+0000","last_undegraded":"2024-09-21T10:18:38.688620+0000","last_fullsized":"2024-09-21T10:18:38.688620+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":301,"reported_epoch":298,"state":"active+undersized","last_fresh":"2024-09-21T10:18:37.686298+0000","last_change":"2024-09-21T10:17:35.127626+0000","last_active":"2024-09-21T10:18:37.686298+0000","last_peered":"2024-09-21T10:18:37.686298+0000","last_clean":"2024-09-21T10:17:25.109952+0000","last_became_active":"2024-09-21T10:17:35.127626+0000","last_became_peered":"2024-09-21T10:17:35.127626+0000","last_unstale":"2024-09-21T10:18:37.686298+0000","last_undegraded":"2024-09-21T10:18:37.686298+0000","last_fullsized":"2024-09-21T10:17:35.121474+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.365796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":496,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.688720+0000","last_change":"2024-09-21T10:13:09.581932+0000","last_active":"2024-09-21T10:18:38.688720+0000","last_peered":"2024-09-21T10:18:38.688720+0000","last_clean":"2024-09-21T10:18:38.688720+0000","last_became_active":"2024-09-21T10:13:09.580798+0000","last_became_peered":"2024-09-21T10:13:09.580798+0000","last_unstale":"2024-09-21T10:18:38.688720+0000","last_undegraded":"2024-09-21T10:18:38.688720+0000","last_fullsized":"2024-09-21T10:18:38.688720+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:05.563826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":78,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.688074+0000","last_change":"2024-09-21T10:18:31.642807+0000","last_active":"2024-09-21T10:18:38.688074+0000","last_peered":"2024-09-21T10:18:38.688074+0000","last_clean":"2024-09-21T10:18:38.688074+0000","last_became_active":"2024-09-21T10:18:31.642455+0000","last_became_peered":"2024-09-21T10:18:31.642455+0000","last_unstale":"2024-09-21T10:18:38.688074+0000","last_undegraded":"2024-09-21T10:18:38.688074+0000","last_fullsized":"2024-09-21T10:18:38.688074+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:29.168508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":187,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.756593+0000","last_change":"2024-09-21T10:14:32.454735+0000","last_active":"2024-09-21T10:18:38.756593+0000","last_peered":"2024-09-21T10:18:38.756593+0000","last_clean":"2024-09-21T10:18:38.756593+0000","last_became_active":"2024-09-21T10:14:32.454475+0000","last_became_peered":"2024-09-21T10:14:32.454475+0000","last_unstale":"2024-09-21T10:18:38.756593+0000","last_undegraded":"2024-09-21T10:18:38.756593+0000","last_fullsized":"2024-09-21T10:18:38.756593+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:59.191649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":478,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.689105+0000","last_change":"2024-09-21T10:14:20.761163+0000","last_active":"2024-09-21T10:18:38.689105+0000","last_peered":"2024-09-21T10:18:38.689105+0000","last_clean":"2024-09-21T10:18:38.689105+0000","last_became_active":"2024-09-21T10:14:20.760814+0000","last_became_peered":"2024-09-21T10:14:20.760814+0000","last_unstale":"2024-09-21T10:18:38.689105+0000","last_undegraded":"2024-09-21T10:18:38.689105+0000","last_fullsized":"2024-09-21T10:18:38.689105+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:44.679743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":301,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.704692+0000","last_change":"2024-09-21T10:13:09.577994+0000","last_active":"2024-09-21T10:18:37.704692+0000","last_peered":"2024-09-21T10:18:37.704692+0000","last_clean":"2024-09-21T10:18:37.704692+0000","last_became_active":"2024-09-21T10:13:09.577311+0000","last_became_peered":"2024-09-21T10:13:09.577311+0000","last_unstale":"2024-09-21T10:18:37.704692+0000","last_undegraded":"2024-09-21T10:18:37.704692+0000","last_fullsized":"2024-09-21T10:18:37.704692+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:27.532202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":492,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.691686+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:18:38.691686+0000","last_peered":"2024-09-21T10:18:38.691686+0000","last_clean":"2024-09-21T10:18:38.691686+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:18:38.691686+0000","last_undegraded":"2024-09-21T10:18:38.691686+0000","last_fullsized":"2024-09-21T10:18:38.691686+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":180,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.706535+0000","last_change":"2024-09-21T10:13:09.578314+0000","last_active":"2024-09-21T10:18:37.706535+0000","last_peered":"2024-09-21T10:18:37.706535+0000","last_clean":"2024-09-21T10:18:37.706535+0000","last_became_active":"2024-09-21T10:13:09.577570+0000","last_became_peered":"2024-09-21T10:13:09.577570+0000","last_unstale":"2024-09-21T10:18:37.706535+0000","last_undegraded":"2024-09-21T10:18:37.706535+0000","last_fullsized":"2024-09-21T10:18:37.706535+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:35:21.299713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":146,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.706449+0000","last_change":"2024-09-21T10:13:09.577841+0000","last_active":"2024-09-21T10:18:37.706449+0000","last_peered":"2024-09-21T10:18:37.706449+0000","last_clean":"2024-09-21T10:18:37.706449+0000","last_became_active":"2024-09-21T10:13:09.576793+0000","last_became_peered":"2024-09-21T10:13:09.576793+0000","last_unstale":"2024-09-21T10:18:37.706449+0000","last_undegraded":"2024-09-21T10:18:37.706449+0000","last_fullsized":"2024-09-21T10:18:37.706449+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:12.718520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":38,"reported_epoch":298,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:37.705340+0000","last_change":"2024-09-21T10:17:35.128563+0000","last_active":"2024-09-21T10:18:37.705340+0000","last_peered":"2024-09-21T10:18:37.705340+0000","last_clean":"2024-09-21T09:50:27.027024+0000","last_became_active":"2024-09-21T10:17:35.128563+0000","last_became_peered":"2024-09-21T10:17:35.128563+0000","last_unstale":"2024-09-21T10:18:37.705340+0000","last_undegraded":"2024-09-21T10:17:35.123737+0000","last_fullsized":"2024-09-21T10:17:35.123629+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:24:08.945613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":71,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.704549+0000","last_change":"2024-09-21T10:16:35.252818+0000","last_active":"2024-09-21T10:18:37.704549+0000","last_peered":"2024-09-21T10:18:37.704549+0000","last_clean":"2024-09-21T10:18:37.704549+0000","last_became_active":"2024-09-21T10:16:35.252497+0000","last_became_peered":"2024-09-21T10:16:35.252497+0000","last_unstale":"2024-09-21T10:18:37.704549+0000","last_undegraded":"2024-09-21T10:18:37.704549+0000","last_fullsized":"2024-09-21T10:18:37.704549+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:07:30.018576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":38,"reported_epoch":298,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:37.705359+0000","last_change":"2024-09-21T10:17:35.124641+0000","last_active":"2024-09-21T10:18:37.705359+0000","last_peered":"2024-09-21T10:18:37.705359+0000","last_clean":"2024-09-21T09:50:27.027148+0000","last_became_active":"2024-09-21T10:17:35.124641+0000","last_became_peered":"2024-09-21T10:17:35.124641+0000","last_unstale":"2024-09-21T10:18:37.705359+0000","last_undegraded":"2024-09-21T10:17:35.122956+0000","last_fullsized":"2024-09-21T10:17:35.122826+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:05.334896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":506,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.688963+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:18:38.688963+0000","last_peered":"2024-09-21T10:18:38.688963+0000","last_clean":"2024-09-21T10:18:38.688963+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:18:38.688963+0000","last_undegraded":"2024-09-21T10:18:38.688963+0000","last_fullsized":"2024-09-21T10:18:38.688963+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":467,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-21T10:18:38.689648+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:18:38.689648+0000","last_peered":"2024-09-21T10:18:38.689648+0000","last_clean":"2024-09-21T10:18:38.689648+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:18:38.689648+0000","last_undegraded":"2024-09-21T10:18:38.689648+0000","last_fullsized":"2024-09-21T10:18:38.689648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":284,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-09-21T10:18:37.704527+0000","last_change":"2024-09-21T10:13:09.577388+0000","last_active":"2024-09-21T10:18:37.704527+0000","last_peered":"2024-09-21T10:18:37.704527+0000","last_clean":"2024-09-21T10:18:37.704527+0000","last_became_active":"2024-09-21T10:13:09.577027+0000","last_became_peered":"2024-09-21T10:13:09.577027+0000","last_unstale":"2024-09-21T10:18:37.704527+0000","last_undegraded":"2024-09-21T10:18:37.704527+0000","last_fullsized":"2024-09-21T10:18:37.704527+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:49.522609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":208,"reported_epoch":299,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:18:38.691913+0000","last_change":"2024-09-21T10:17:35.129614+0000","last_active":"2024-09-21T10:18:38.691913+0000","last_peered":"2024-09-21T10:18:38.691913+0000","last_clean":"2024-09-21T10:14:43.015500+0000","last_became_active":"2024-09-21T10:17:35.129614+0000","last_became_peered":"2024-09-21T10:17:35.129614+0000","last_unstale":"2024-09-21T10:18:38.691913+0000","last_undegraded":"2024-09-21T10:17:35.124679+0000","last_fullsized":"2024-09-21T10:17:35.124462+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:00:36.906750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":291897368,"num_objects":4455,"num_object_clones":0,"num_object_copies":13365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1026,"num_objects_misplaced":316,"num_objects_unfound":0,"num_objects_dirty":4455,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":244873,"num_write_kb":7978433,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14764,"num_bytes_recovered":967114776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2178625536,"data_stored":2178613320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108590,"ondisk_log_size":108590,"up":43,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":84,"acting":90,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":812,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18547,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":81,"acting":86,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":170,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":22,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234348,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":696156,"kb_used_data":173648,"kb_used_omap":1,"kb_used_meta":522494,"kb_avail":93048996,"statfs":{"total":95995035648,"available":95282171904,"internally_reserved":0,"allocated":177815552,"data_stored":176415320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":535034315},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495986,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":629292,"kb_used_data":187164,"kb_used_omap":1,"kb_used_meta":442110,"kb_avail":93115860,"statfs":{"total":95995035648,"available":95350640640,"internally_reserved":0,"allocated":191655936,"data_stored":190244860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":452721095},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757609,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1670740,"kb_used_data":1274572,"kb_used_omap":1,"kb_used_meta":396158,"kb_avail":92074412,"statfs":{"total":95995035648,"available":94284197888,"internally_reserved":0,"allocated":1305161728,"data_stored":1303858859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":405666246},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019259,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":615012,"kb_used_data":168788,"kb_used_omap":1,"kb_used_meta":446206,"kb_avail":93130140,"statfs":{"total":95995035648,"available":95365263360,"internally_reserved":0,"allocated":172838912,"data_stored":171434935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":456915398},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435444,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":712776,"kb_used_data":139648,"kb_used_omap":5,"kb_used_meta":573114,"kb_avail":93032376,"statfs":{"total":95995035648,"available":95265153024,"internally_reserved":0,"allocated":142999552,"data_stored":141611255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":586868776},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598971,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":637132,"kb_used_data":64004,"kb_used_omap":7,"kb_used_meta":573112,"kb_avail":93108020,"statfs":{"total":95995035648,"available":95342612480,"internally_reserved":0,"allocated":65540096,"data_stored":64148465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":586867149},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762497,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":617624,"kb_used_data":184648,"kb_used_omap":7,"kb_used_meta":432888,"kb_avail":93127528,"statfs":{"total":95995035648,"available":95362588672,"internally_reserved":0,"allocated":189079552,"data_stored":187683309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":443277565},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926024,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416524,"kb_used_data":64132,"kb_used_omap":7,"kb_used_meta":352376,"kb_avail":93328628,"statfs":{"total":95995035648,"available":95568515072,"internally_reserved":0,"allocated":65671168,"data_stored":64278799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":360833214},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57544704,"data_stored":57540632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":166596608,"data_stored":166592536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":52363264,"data_stored":52363264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":109314048,"data_stored":109314048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":156237824,"data_stored":156237824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1295253504,"data_stored":1295253504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":180555776,"data_stored":180551704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":160759808,"data_stored":160759808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:18:47.607 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 11 2024-09-21T10:18:47.640 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:47.673 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:47.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:47.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:47.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:47.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:47.829 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:47 smithi032 ceph-mon[101299]: from='client.45428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:18:47.830 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:47 smithi032 ceph-mon[101299]: pgmap v886: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 131 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 950 op/s; 456/6159 objects degraded (7.404%); 321/6159 objects misplaced (5.212%); 3.5 MiB/s, 56 objects/s recovering 2024-09-21T10:18:47.830 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:47 smithi032 ceph-mon[105748]: from='client.45428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:18:47.830 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:47 smithi032 ceph-mon[105748]: pgmap v886: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 131 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 950 op/s; 456/6159 objects degraded (7.404%); 321/6159 objects misplaced (5.212%); 3.5 MiB/s, 56 objects/s recovering 2024-09-21T10:18:47.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:47.922 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:47.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:48.023 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:47 smithi045 ceph-mon[95976]: from='client.45428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:18:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:47 smithi045 ceph-mon[95976]: pgmap v886: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 131 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 950 op/s; 456/6159 objects degraded (7.404%); 321/6159 objects misplaced (5.212%); 3.5 MiB/s, 56 objects/s recovering 2024-09-21T10:18:48.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:48.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:48.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:48.170 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:48.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:48.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:48.299 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:48.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:48.401 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.425 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:48.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:48.526 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:48.552 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:48.654 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:48.680 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:48.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:48.781 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:48.804 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:48.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:48.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:48.927 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:48.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:49.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:49.051 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:49.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:49.153 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:49.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:49.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:49.277 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:49.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:49.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:49.405 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:49.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:49.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:49.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:49.554 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:49.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:49.595 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:49.595 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:49.634 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:49 smithi032 ceph-mon[101299]: pgmap v887: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 111 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 456/5211 objects degraded (8.751%); 5/5211 objects misplaced (0.096%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:18:49.634 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:49.634 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:49.635 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:18:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:18:49] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.51.0" 2024-09-21T10:18:49.635 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:49 smithi032 ceph-mon[105748]: pgmap v887: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 111 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 456/5211 objects degraded (8.751%); 5/5211 objects misplaced (0.096%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:18:49.635 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:49.635 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:49.656 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:49.682 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:49.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:49.784 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:49 smithi045 ceph-mon[95976]: pgmap v887: 99 pgs: 13 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 65 active+clean; 111 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 456/5211 objects degraded (8.751%); 5/5211 objects misplaced (0.096%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:18:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:18:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:49.810 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:49.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:49.878 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:18:49.878 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:18:49.879 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 0 2024-09-21T10:18:49.879 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:18:49.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:49.935 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:50.037 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:50.059 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:50.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:50.188 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:50.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:50.290 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:50.318 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:50.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:50.419 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:50.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:50.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:50.500 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:18:50.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:50.574 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:50.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:50.676 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:50.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:50.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:50.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:50 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3612654203' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:50.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:50 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:50.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:50 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3612654203' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:50.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:50 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:50.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:50 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3612654203' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:50.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:50 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:50.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:50.823 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:50.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:50.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:50.952 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:50.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:51.054 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:51.080 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:51.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:51.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:51.204 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:51.306 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:51.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:51.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:51.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:51.456 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:51.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:51.584 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:51.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:51.713 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:51.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:51 smithi045 ceph-mon[95976]: pgmap v888: 99 pgs: 13 active+clean+remapped, 13 active+undersized, 8 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:18:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:51 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:51 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:51.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:51 smithi045 ceph-mon[95976]: osdmap e300: 8 total, 7 up, 8 in 2024-09-21T10:18:51.814 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:51.840 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:51.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:51.943 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:51 smithi032 ceph-mon[101299]: pgmap v888: 99 pgs: 13 active+clean+remapped, 13 active+undersized, 8 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:18:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:51 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:51 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:51 smithi032 ceph-mon[101299]: osdmap e300: 8 total, 7 up, 8 in 2024-09-21T10:18:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:51 smithi032 ceph-mon[105748]: pgmap v888: 99 pgs: 13 active+clean+remapped, 13 active+undersized, 8 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:18:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:51 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:51 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:51 smithi032 ceph-mon[105748]: osdmap e300: 8 total, 7 up, 8 in 2024-09-21T10:18:51.970 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:51.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:52.071 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:52.097 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:52.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:52.199 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:52.226 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:52.327 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:52.353 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:52.455 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:52.482 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:52.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:52.584 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:52.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:52.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:52.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:52.736 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:52.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:52 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 8 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:52.838 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:52.864 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:52.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:52 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 8 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:52.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:52 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 8 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:52.965 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:52.992 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:52.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:53.094 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:53.120 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:53.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:53.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:53.249 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:53.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:53.350 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:53.378 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:53.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:53.479 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:53.506 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:53.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:53.607 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:53.634 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:53.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:53.735 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:53.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.762 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:53.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:53.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:53 smithi045 ceph-mon[95976]: pgmap v890: 99 pgs: 13 active+clean+remapped, 13 active+undersized, 8 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:53.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:53 smithi032 ceph-mon[101299]: pgmap v890: 99 pgs: 13 active+clean+remapped, 13 active+undersized, 8 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:53.845 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:53 smithi032 ceph-mon[105748]: pgmap v890: 99 pgs: 13 active+clean+remapped, 13 active+undersized, 8 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:53.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:53.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:53.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:53.989 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:54.014 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:54.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:54.116 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:54.141 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:54.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:54.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:54.266 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:54.368 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:54.395 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:54.497 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:54.524 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:54.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:54.626 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:54.650 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:54.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:54.752 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:54.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:54.877 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:54.903 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:54.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:55.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:55.028 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:55.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:55.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:55.152 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:55.254 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:55.277 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:55.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:55.378 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:55.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:18:55.405 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:55.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:55.506 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:55.530 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:55.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:55.631 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:55.644 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:55 smithi032 ceph-mon[105748]: pgmap v891: 99 pgs: 13 active+clean+remapped, 13 active+undersized, 8 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:55.644 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:55 smithi032 ceph-mon[101299]: pgmap v891: 99 pgs: 13 active+clean+remapped, 13 active+undersized, 8 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:55.658 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:55.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:55.760 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:55.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:55 smithi045 ceph-mon[95976]: pgmap v891: 99 pgs: 13 active+clean+remapped, 13 active+undersized, 8 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:55.787 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:55.877 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:18:55.877 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:18:55.889 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:55.894 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:18:55.894 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:55.910 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:18:55.910 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:55.915 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:55.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:55.927 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:18:55.927 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:55.944 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:18:55.944 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:55.944 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:18:55.960 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:18:55.961 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:55.977 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:18:55.977 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:56.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:56.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.041 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:56.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:56.142 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:56.165 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:56.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:56.267 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:56.288 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:56.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:56.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:56.417 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:56.422 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:56.515 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 1 pgp_num to 11 2024-09-21T10:18:56.519 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:56.543 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:56.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:18:56.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:56.671 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:56.773 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:56.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/468187938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:56.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/935184070' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:18:56.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:56 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:18:56.795 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:56.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:56.897 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:56.918 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/468187938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/935184070' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:18:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:56 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:18:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/468187938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/935184070' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:18:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:56 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:18:57.003 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:18:57.004 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:18:57.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:57.068 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:57.170 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:57.192 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:57.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:57.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:57.317 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:57.351 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:57.419 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:57.441 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:57.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:57.543 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:57.565 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:57.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:57.667 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:57.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.786 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:57.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:57.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:57 smithi045 ceph-mon[95976]: pgmap v892: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:57.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:57 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T10:18:57.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:57 smithi045 ceph-mon[95976]: osdmap e301: 8 total, 7 up, 8 in 2024-09-21T10:18:57.787 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:57 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:57.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:57 smithi032 ceph-mon[105748]: pgmap v892: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:57.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:57 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T10:18:57.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:57 smithi032 ceph-mon[105748]: osdmap e301: 8 total, 7 up, 8 in 2024-09-21T10:18:57.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:57 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:57.805 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:57 smithi032 ceph-mon[101299]: pgmap v892: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:57.805 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:57 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T10:18:57.805 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:57 smithi032 ceph-mon[101299]: osdmap e301: 8 total, 7 up, 8 in 2024-09-21T10:18:57.805 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:57 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:18:57.887 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.917 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:57.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:58.018 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:58.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:58.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:58.147 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:58.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:58.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:58.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:58.300 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:58.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:58.401 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:58.427 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:58.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:58.529 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:58.533 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-09-21T10:18:58.533 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-09-21T10:18:58.554 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:58.656 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:58.681 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:58.783 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:58.806 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:58.908 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:58.930 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:59.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:18:59.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:59.069 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:59.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:59.171 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:59.194 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:59.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:59.297 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:59.319 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:59.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:59.421 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.450 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:59.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:59.535 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:18:59 smithi032 ceph-mon[101299]: pgmap v894: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:59.535 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:18:59 smithi032 ceph-mon[105748]: pgmap v894: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:59.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:59.574 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:59.676 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:59.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:59.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:18:59 smithi045 ceph-mon[95976]: pgmap v894: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:18:59.800 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:59.812 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:18:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:18:59] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.51.0" 2024-09-21T10:18:59.827 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:18:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:59.929 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:59.953 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:18:59.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:00.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:00.077 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:00.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:00.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:00.200 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:00.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:00.232 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:00.302 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:00.325 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:00.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:00.449 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:00.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:00.525 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:19:00.551 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:00.578 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:00.679 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:00.702 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:00.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:00.704 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:00 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/562275416' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:00.704 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:00.704 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:00 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/562275416' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:00.705 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:00.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:00 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/562275416' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:00.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:00.804 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:00.831 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:00.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:00.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:00.957 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:01.059 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:01.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.087 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:01.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:01.188 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:01.214 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:01.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:01.316 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:01.339 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:01.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:01.441 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:01.468 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:01.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:01.569 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.593 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:01.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:01.694 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:01.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:01 smithi045 ceph-mon[95976]: pgmap v895: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:01 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:01 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:19:01.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:01 smithi045 ceph-mon[95976]: osdmap e302: 8 total, 7 up, 8 in 2024-09-21T10:19:01.787 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:01 smithi032 ceph-mon[105748]: pgmap v895: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:01.788 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:01 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:01.788 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:01 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:19:01.788 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:01 smithi032 ceph-mon[105748]: osdmap e302: 8 total, 7 up, 8 in 2024-09-21T10:19:01.788 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:01 smithi032 ceph-mon[101299]: pgmap v895: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:01.788 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:01 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:01.788 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:01 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:19:01.788 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:01 smithi032 ceph-mon[101299]: osdmap e302: 8 total, 7 up, 8 in 2024-09-21T10:19:01.818 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:01.841 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:01.920 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:19:01.921 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:19:01.942 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:01.965 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:01.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:02.066 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:02.093 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:02.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:02.221 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:02.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:02.323 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:02.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:02.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:02.448 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:02.475 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:02.577 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:02.599 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:02.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:02.633 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:19:02.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:02.704 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:02.728 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:02.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:02.830 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:02.858 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:02.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:02.959 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:02.985 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:03.087 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:03.108 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:03.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:03.210 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:03.237 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:03.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:03.338 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:03.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:03.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:03.463 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:03.484 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:03.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:03.534 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-09-21T10:19:03.534 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:19:03.534 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-21T10:19:03.534 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:19:03.534 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:19:03.534 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-21T10:19:03.586 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:03.608 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:03.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:19:03.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:03.732 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:03.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:19:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:03 smithi045 ceph-mon[95976]: 1.4 scrub starts 2024-09-21T10:19:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:03 smithi045 ceph-mon[95976]: 1.4 scrub ok 2024-09-21T10:19:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:03 smithi045 ceph-mon[95976]: pgmap v897: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:03.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:03.833 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:03.851 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:03 smithi032 ceph-mon[105748]: 1.4 scrub starts 2024-09-21T10:19:03.851 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:03 smithi032 ceph-mon[105748]: 1.4 scrub ok 2024-09-21T10:19:03.851 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:03 smithi032 ceph-mon[105748]: pgmap v897: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:03.851 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:03.851 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:03 smithi032 ceph-mon[101299]: 1.4 scrub starts 2024-09-21T10:19:03.851 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:03 smithi032 ceph-mon[101299]: 1.4 scrub ok 2024-09-21T10:19:03.851 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:03 smithi032 ceph-mon[101299]: pgmap v897: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:03.852 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:03.856 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:03.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:03.957 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:03.980 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:03.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:04.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.106 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:04.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:04.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:04.230 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:04.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:04.331 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:04.353 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:04.455 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:04.479 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:04.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:04.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:04.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:04.704 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:04.726 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:04.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:04.828 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:04.850 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:04.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:04.952 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:04.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:04.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:05.076 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:05.099 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:05.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:05.201 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:05.224 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:05.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:05.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:05.342 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:19:05.343 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:19:05.343 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:05.343 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:19:05.349 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:05.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:05.354 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:19:05.354 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:05.370 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:19:05.370 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:05.386 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:19:05.387 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:05.387 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:19:05.403 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:19:05.403 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:05.421 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:19:05.421 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:05.451 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:05.464 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:19:05.464 INFO:teuthology.orchestra.run.smithi045.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:19:05.464 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:19:05.473 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:05.574 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:05.596 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:05.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:05.698 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:05 smithi032 ceph-mon[105748]: 1.7 scrub starts 2024-09-21T10:19:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:05 smithi032 ceph-mon[105748]: 1.7 scrub ok 2024-09-21T10:19:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:05 smithi032 ceph-mon[105748]: pgmap v898: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:05.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1671480201' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:05 smithi032 ceph-mon[101299]: 1.7 scrub starts 2024-09-21T10:19:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:05 smithi032 ceph-mon[101299]: 1.7 scrub ok 2024-09-21T10:19:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:05 smithi032 ceph-mon[101299]: pgmap v898: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:05.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1671480201' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:05.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:05 smithi045 ceph-mon[95976]: 1.7 scrub starts 2024-09-21T10:19:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:05 smithi045 ceph-mon[95976]: 1.7 scrub ok 2024-09-21T10:19:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:05 smithi045 ceph-mon[95976]: pgmap v898: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:05.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1671480201' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:05.823 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:05.845 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:05.946 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:05.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:05.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:06.070 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:06.095 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:06.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:06.106 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:06.142 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:06.196 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:06.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:06.321 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:06.344 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:06.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:06.445 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:06.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:06.570 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:06.593 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:06.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:06.649 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:06 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10/87 objects degraded (11.494%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:06.649 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:06 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10/87 objects degraded (11.494%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:06.695 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:06.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:06.823 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:06.845 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:06.946 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:06.973 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:06.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:06 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10/87 objects degraded (11.494%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:07.074 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:07.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:07.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:07.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:07.220 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:07.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:07.322 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:07.348 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:07.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:07.450 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:19:07.460 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:19:07.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:07.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:07.578 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:07.584 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_2' removed 2024-09-21T10:19:07.619 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:07.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:07.721 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:07.743 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:07.845 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:07.867 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:07.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:07.928 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:07 smithi032 ceph-mon[101299]: pgmap v899: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.8 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:07.929 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:07 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/422849520' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:19:07.929 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:07 smithi032 ceph-mon[105748]: pgmap v899: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.8 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:07.929 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:07 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/422849520' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:19:07.968 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:08.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.005 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:08.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:07 smithi045 ceph-mon[95976]: pgmap v899: 99 pgs: 13 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 65 active+clean; 4.8 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:07 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/422849520' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:19:08.106 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:08.129 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:08.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:08.230 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:08.254 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:08.356 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:08.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:08.480 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:08.506 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:08.607 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:08.630 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:08.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:08.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:08 smithi032 ceph-mon[101299]: 1.6 scrub starts 2024-09-21T10:19:08.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:08 smithi032 ceph-mon[101299]: 1.6 scrub ok 2024-09-21T10:19:08.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:08 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/422849520' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:19:08.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:08 smithi032 ceph-mon[101299]: osdmap e303: 8 total, 7 up, 8 in 2024-09-21T10:19:08.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:08 smithi032 ceph-mon[105748]: 1.6 scrub starts 2024-09-21T10:19:08.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:08 smithi032 ceph-mon[105748]: 1.6 scrub ok 2024-09-21T10:19:08.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:08 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/422849520' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:19:08.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:08 smithi032 ceph-mon[105748]: osdmap e303: 8 total, 7 up, 8 in 2024-09-21T10:19:08.732 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:08.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:08.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:08.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:08.878 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:08.979 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:09.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.002 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:09.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:09.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:08 smithi045 ceph-mon[95976]: 1.6 scrub starts 2024-09-21T10:19:09.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:08 smithi045 ceph-mon[95976]: 1.6 scrub ok 2024-09-21T10:19:09.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:08 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/422849520' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:19:09.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:08 smithi045 ceph-mon[95976]: osdmap e303: 8 total, 7 up, 8 in 2024-09-21T10:19:09.102 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:09.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:09.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:09.231 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:09.257 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:09.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:09.359 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:09.384 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:09.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:09.461 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:19:09.485 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:09.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:09.627 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:09.649 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:09.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:09.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:19:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:19:09] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-21T10:19:09.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:09 smithi032 ceph-mon[101299]: 1.5 scrub starts 2024-09-21T10:19:09.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:09 smithi032 ceph-mon[101299]: 1.5 scrub ok 2024-09-21T10:19:09.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:09 smithi032 ceph-mon[101299]: pgmap v901: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.8 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:09.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:09 smithi032 ceph-mon[105748]: 1.5 scrub starts 2024-09-21T10:19:09.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:09 smithi032 ceph-mon[105748]: 1.5 scrub ok 2024-09-21T10:19:09.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:09 smithi032 ceph-mon[105748]: pgmap v901: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.8 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:09.751 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:09.779 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:09.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:09.826 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:19:09.827 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:19:09.827 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-09-21T10:19:09.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_3 16 2024-09-21T10:19:09.881 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:09.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:09.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:10.005 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:10.032 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:10.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:09 smithi045 ceph-mon[95976]: 1.5 scrub starts 2024-09-21T10:19:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:09 smithi045 ceph-mon[95976]: 1.5 scrub ok 2024-09-21T10:19:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:09 smithi045 ceph-mon[95976]: pgmap v901: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.8 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:10.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:10.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:10.258 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:10.284 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:10.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:10.386 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:10.413 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:10.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:10.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:10.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:10.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:10.642 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:10.667 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:10.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:10.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:10.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:10.896 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:10.923 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:10.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:11.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:11.048 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:11.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:11.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:11.153 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241767 5.b 5:dd8b8b4d:::benchmark_data_smithi032_140417_object108376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.559871+0000", 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.58848569400004, 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.97648648699999996, 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.154 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241767 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559871+0000", 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.155 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559871+0000", 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559873+0000", 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9580000000000001e-06 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559879+0000", 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1490000000000002e-06 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.156 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559891+0000", 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2670999999999999e-05 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559899+0000", 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0600000000000008e-06 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.559984+0000", 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4837000000000001e-05 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.157 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.560121+0000", 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013717499999999999 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5,7", 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.560241+0000", 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000119852 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.567391+0000", 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0071502320000000003 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.158 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.568018+0000", 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00062714499999999996 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.577631+0000", 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0096120939999999998 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585548+0000", 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0079174420000000002 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.159 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585561+0000", 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3477e-05 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536278+0000", 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95071670500000005 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536358+0000", 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9690000000000004e-05 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.160 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241780 5.b 5:dbce6705:::benchmark_data_smithi032_140417_object108389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.578234+0000", 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.57012283799997, 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95871411699999998, 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241780 2024-09-21T10:19:11.161 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578234+0000", 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578234+0000", 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578236+0000", 2024-09-21T10:19:11.162 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8279999999999999e-06 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578241+0000", 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3630000000000002e-06 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578258+0000", 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6198999999999999e-05 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578266+0000", 2024-09-21T10:19:11.163 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1899999999999995e-06 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578403+0000", 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013736600000000001 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578564+0000", 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016075700000000001 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5,7", 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.578692+0000", 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000127663 2024-09-21T10:19:11.164 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585713+0000", 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0070217600000000002 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585722+0000", 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4800000000000001e-06 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536878+0000", 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95115663500000003 2024-09-21T10:19:11.165 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536948+0000", 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9876000000000007e-05 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241783 5.b 5:d9813225:::benchmark_data_smithi032_140417_object108392:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.583475+0000", 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.56488184099999, 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.96363426399999996, 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.166 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241783 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583475+0000", 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.167 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583475+0000", 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583477+0000", 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2069999999999998e-06 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583483+0000", 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.835e-06 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583500+0000", 2024-09-21T10:19:11.168 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6920999999999999e-05 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.583508+0000", 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6650000000000003e-06 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585752+0000", 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0022445170000000001 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.585760+0000", 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9240000000000007e-06 2024-09-21T10:19:11.169 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.536976+0000", 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95121591100000003 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.537083+0000", 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000107181 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.537170+0000", 2024-09-21T10:19:11.170 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6984000000000006e-05 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.546792+0000", 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0096215229999999999 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.546849+0000", 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7135000000000001e-05 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.546981+0000", 2024-09-21T10:19:11.171 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013146699999999999 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547069+0000", 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8830000000000002e-05 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547110+0000", 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0163999999999997e-05 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.172 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241790 5.b 5:d9f071b7:::benchmark_data_smithi032_140417_object108399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.588776+0000", 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.55958122899995, 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.98105678799999996, 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241790 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.173 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588776+0000", 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588776+0000", 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588777+0000", 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0750000000000001e-06 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.174 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588781+0000", 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4530000000000004e-06 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588796+0000", 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4842e-05 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588804+0000", 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9899999999999997e-06 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.175 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588920+0000", 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011530400000000001 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.588930+0000", 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0526999999999999e-05 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.542310+0000", 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.95338037600000003 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.176 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.542440+0000", 2024-09-21T10:19:11.177 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000129794 2024-09-21T10:19:11.177 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.177 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.177 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:11.177 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.542551+0000", 2024-09-21T10:19:11.178 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000110707 2024-09-21T10:19:11.178 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.178 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.178 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.178 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.546886+0000", 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0043350580000000001 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560416+0000", 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013530333 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569756+0000", 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0093396910000000007 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569791+0000", 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4666999999999999e-05 2024-09-21T10:19:11.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569833+0000", 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1971000000000002e-05 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.180 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241804 5.b 5:d56f6689:::benchmark_data_smithi032_140417_object108413:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:19:11.180 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-09-21T10:19:11.180 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:19:11.180 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-21T10:19:11.181 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:19:11.181 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:19:11.181 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.604378+0000", 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.54397944000004, 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.965583466, 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241804 2024-09-21T10:19:11.182 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604378+0000", 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604378+0000", 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604379+0000", 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.207e-06 2024-09-21T10:19:11.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604384+0000", 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1599999999999997e-06 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604394+0000", 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4900000000000006e-06 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604400+0000", 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2310000000000001e-06 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604454+0000", 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3767000000000002e-05 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.604462+0000", 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1610000000000003e-06 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547153+0000", 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.94269127799999997 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.185 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547257+0000", 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010452 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.547378+0000", 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000120383 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560454+0000", 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013076119000000001 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.186 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560492+0000", 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8175e-05 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569891+0000", 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0093986539999999993 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569939+0000", 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7734000000000001e-05 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.187 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569961+0000", 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2586999999999999e-05 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241832 5.b 5:dd944995:::benchmark_data_smithi032_140417_object108441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.636813+0000", 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.51154363700005, 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.93322067200000003, 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.188 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241832 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636813+0000", 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636813+0000", 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.189 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636815+0000", 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3859999999999999e-06 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636820+0000", 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9359999999999998e-06 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636836+0000", 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6711e-05 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.636845+0000", 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0560000000000005e-06 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.637012+0000", 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016762100000000001 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.637022+0000", 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3950000000000007e-06 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.551757+0000", 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.91473561999999997 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.551886+0000", 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012900699999999999 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:11.192 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.552039+0000", 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000152597 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560541+0000", 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0085025929999999993 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569747+0000", 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0092056529999999994 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.193 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569970+0000", 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000222554 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570012+0000", 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2849000000000003e-05 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570034+0000", 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1693999999999999e-05 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.194 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241877 5.b 5:d8095acf:::benchmark_data_smithi032_140417_object108486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.682983+0000", 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.46537400800003, 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.88718399199999998, 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241877 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.195 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.682983+0000", 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.682983+0000", 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.682984+0000", 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8999999999999995e-07 2024-09-21T10:19:11.196 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.682987+0000", 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.569e-06 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.683033+0000", 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5096000000000003e-05 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.683042+0000", 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2639999999999997e-06 2024-09-21T10:19:11.197 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.684225+0000", 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001183403 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.684234+0000", 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5529999999999993e-06 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.555793+0000", 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.87155956400000001 2024-09-21T10:19:11.198 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.555895+0000", 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000101887 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.556014+0000", 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000118559 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560577+0000", 2024-09-21T10:19:11.199 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.004563587 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570044+0000", 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0094666330000000003 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570101+0000", 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6842999999999998e-05 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570130+0000", 2024-09-21T10:19:11.200 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.9115000000000002e-05 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570167+0000", 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.7129000000000003e-05 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241888 5.b 5:dd26de34:::benchmark_data_smithi032_140417_object108497:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.695983+0000", 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.45237413799998, 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.88328308200000005, 2024-09-21T10:19:11.201 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241888 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.695983+0000", 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.202 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.695983+0000", 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.695984+0000", 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2759999999999999e-06 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.695989+0000", 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0289999999999996e-06 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.203 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.696000+0000", 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0343000000000001e-05 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.696007+0000", 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3010000000000001e-06 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.204 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.205 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.696305+0000", 2024-09-21T10:19:11.205 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029825899999999999 2024-09-21T10:19:11.205 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.205 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.205 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.205 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.696314+0000", 2024-09-21T10:19:11.205 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3909999999999998e-06 2024-09-21T10:19:11.205 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560605+0000", 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.86429099799999998 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560720+0000", 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011551499999999999 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.560837+0000", 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011686 2024-09-21T10:19:11.206 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.569879+0000", 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0090424839999999999 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570052+0000", 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000172614 2024-09-21T10:19:11.207 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579189+0000", 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0091371950000000007 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579222+0000", 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2437000000000001e-05 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579266+0000", 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4379999999999999e-05 2024-09-21T10:19:11.208 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241903 5.b 5:d931b21c:::benchmark_data_smithi032_140417_object108512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.712698+0000", 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.43565890100001, 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.886554913, 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.209 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241903 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712698+0000", 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712698+0000", 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.210 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712700+0000", 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.477e-06 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712704+0000", 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4549999999999997e-06 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712796+0000", 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1836999999999995e-05 2024-09-21T10:19:11.211 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712806+0000", 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0049e-05 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712907+0000", 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000101099 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.712916+0000", 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9979999999999992e-06 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.212 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.565163+0000", 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.85224674199999995 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.565269+0000", 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010645900000000001 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.565386+0000", 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000116946 2024-09-21T10:19:11.213 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570225+0000", 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0048383999999999996 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570242+0000", 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7431e-05 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599133+0000", 2024-09-21T10:19:11.214 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.028890592 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599193+0000", 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9954000000000002e-05 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599253+0000", 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0473999999999997e-05 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.215 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241908 5.b 5:d4d086d0:::benchmark_data_smithi032_140417_object108517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.717789+0000", 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.43056762599997, 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.88166062499999998, 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241908 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.216 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717789+0000", 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717789+0000", 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717791+0000", 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5939999999999999e-06 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.217 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717795+0000", 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8850000000000001e-06 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717806+0000", 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0655999999999999e-05 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717813+0000", 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3189999999999997e-06 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.218 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717870+0000", 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6643e-05 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.717876+0000", 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9780000000000002e-06 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570288+0000", 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.85241283000000001 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.219 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570403+0000", 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114885 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.570518+0000", 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114623 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579324+0000", 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0088064619999999993 2024-09-21T10:19:11.220 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.589427+0000", 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010103031 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599354+0000", 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0099262310000000006 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599411+0000", 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7117999999999998e-05 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.221 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:11.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599450+0000", 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9369999999999997e-05 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44543.0:241909 5.b 5:d4aa5536:::benchmark_data_smithi032_140417_object108518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:09:15.722478+0000", 2024-09-21T10:19:11.224 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 595.42587876100004, 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.87717801399999995, 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44543", 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1338530639", 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 241909 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722478+0000", 2024-09-21T10:19:11.225 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722478+0000", 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722480+0000", 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.172e-06 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.226 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722485+0000", 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1379999999999998e-06 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722495+0000", 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0641e-05 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722503+0000", 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6070000000000002e-06 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.227 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722559+0000", 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6514999999999997e-05 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:15.722568+0000", 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9560000000000003e-06 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.574779+0000", 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.85221065900000004 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.228 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.574885+0000", 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010624300000000001 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.575000+0000", 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114509 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.579378+0000", 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0043779769999999999 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.229 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.589496+0000", 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010117917000000001 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599561+0000", 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010065362 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599615+0000", 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4311999999999998e-05 2024-09-21T10:19:11.230 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:09:16.599656+0000", 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1006e-05 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2390 7.e 7:779d9463:::benchmark_data_smithi032_200324_object2389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.518119+0000", 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 266.63023776599999, 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2855799640000001, 2024-09-21T10:19:11.231 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2390 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518119+0000", 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.232 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518119+0000", 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518120+0000", 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8999999999999996e-07 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518141+0000", 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0672000000000001e-05 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.233 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518149+0000", 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1300000000000001e-06 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518155+0000", 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2689999999999996e-06 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518860+0000", 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00070478499999999998 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.234 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518966+0000", 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000105884 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.519088+0000", 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012275599999999999 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.520196+0000", 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0011078419999999999 2024-09-21T10:19:11.235 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.522225+0000", 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0020289520000000001 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.529365+0000", 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0071399749999999998 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.529379+0000", 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3942e-05 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.236 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524762+0000", 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.99538305699999996 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524777+0000", 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4445e-05 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799459+0000", 2024-09-21T10:19:11.237 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.274682691 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for degraded object", 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800159+0000", 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00069992299999999997 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.803537+0000", 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0033782030000000002 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.238 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.803699+0000", 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000161848 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2417 7.e 7:7656a080:::benchmark_data_smithi032_200324_object2416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.538128+0000", 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 266.61022905099998, 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2674917780000001, 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.239 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2417 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.538128+0000", 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.538128+0000", 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.240 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.538130+0000", 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2989999999999999e-06 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.538164+0000", 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.3602000000000001e-05 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.538184+0000", 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0027999999999999e-05 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.241 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.538193+0000", 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3810000000000005e-06 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.538320+0000", 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012644699999999999 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.538332+0000", 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2003e-05 2024-09-21T10:19:11.242 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524809+0000", 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.98647676900000003 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524817+0000", 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0839999999999992e-06 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800729+0000", 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.275912658 2024-09-21T10:19:11.243 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800865+0000", 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000135976 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801306+0000", 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00044083099999999998 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.802585+0000", 2024-09-21T10:19:11.244 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001279002 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.803742+0000", 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0011570249999999999 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.805049+0000", 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001307204 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.245 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.805370+0000", 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00032037500000000002 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.805620+0000", 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00025009400000000002 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2421 7.e 7:71d011ed:::benchmark_data_smithi032_200324_object2420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.539160+0000", 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 266.60919754499997, 2024-09-21T10:19:11.246 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2668049080000001, 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2421 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.539160+0000", 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.247 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.539160+0000", 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.539161+0000", 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0550000000000001e-06 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.539180+0000", 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8941e-05 2024-09-21T10:19:11.248 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.539194+0000", 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4748e-05 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.539201+0000", 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0509999999999999e-06 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.539276+0000", 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5099999999999996e-05 2024-09-21T10:19:11.249 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.539286+0000", 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5130000000000002e-06 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524843+0000", 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.98555722800000001 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.250 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524851+0000", 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3980000000000002e-06 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801789+0000", 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2769383109999999 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801964+0000", 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017516600000000001 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.251 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.802155+0000", 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000191368 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.803426+0000", 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001270265 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.803790+0000", 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00036434500000000002 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.252 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.805672+0000", 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001882056 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.805724+0000", 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2321999999999998e-05 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.805964+0000", 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000240041 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.253 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2435 7.e 7:781ad222:::benchmark_data_smithi032_200324_object2434:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.544389+0000", 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 266.60396772399997, 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2629164419999999, 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:11.254 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2435 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.544389+0000", 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.544389+0000", 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.255 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.544391+0000", 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5409999999999999e-06 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.544414+0000", 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3377999999999999e-05 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.544430+0000", 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6019e-05 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.256 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.544438+0000", 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5719999999999997e-06 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.544500+0000", 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2355999999999997e-05 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.544511+0000", 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0813e-05 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.257 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524878+0000", 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.98036667200000005 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.524886+0000", 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8490000000000006e-06 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.803830+0000", 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2789447300000001 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.258 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.803985+0000", 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015487300000000001 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.804157+0000", 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017191999999999999 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.804896+0000", 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00073879700000000002 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.259 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.806028+0000", 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001132079 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.806997+0000", 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00096939900000000002 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.807059+0000", 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1894000000000001e-05 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.260 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.807306+0000", 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024655 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:244640 7.e 7:7a34854c:::benchmark_data_smithi032_200324_object119974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:41.511457+0000", 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 29.636900281999999, 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.053034848000000002, 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.261 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 244640 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.511457+0000", 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.511457+0000", 2024-09-21T10:19:11.262 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.511458+0000", 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4300000000000002e-07 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.511462+0000", 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1899999999999997e-06 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.263 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.511504+0000", 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2259999999999999e-05 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.511513+0000", 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6160000000000002e-06 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.524530+0000", 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013017249 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.264 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.524754+0000", 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00022384 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3", 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.524905+0000", 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015162699999999999 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.543484+0000", 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.018578809000000002 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.265 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.564308+0000", 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.020823746000000001 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.564388+0000", 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0105000000000006e-05 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:41.564492+0000", 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000103563 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.266 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:246582 7.e 7:7de26fb8:::benchmark_data_smithi032_200324_object121916:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:18:43.472385+0000", 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 27.675972066, 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.054821903999999998, 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:11.267 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246582 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.472385+0000", 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.472385+0000", 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.268 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.472386+0000", 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9100000000000003e-07 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.472390+0000", 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2930000000000002e-06 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.472401+0000", 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.092e-05 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.269 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.472409+0000", 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0150000000000003e-06 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.480664+0000", 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0082548220000000002 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.481731+0000", 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001066932 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.270 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3", 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.482833+0000", 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0011018639999999999 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.487181+0000", 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0043480100000000002 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.527016+0000", 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.039835679999999998 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.271 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.527096+0000", 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0144999999999999e-05 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:18:43.527207+0000", 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000110432 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "MOSDScrubReserve(1.6,REQUEST,e:302,from:6,reservation_nonce:0)", 2024-09-21T10:19:11.272 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:07.330715+0000", 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 3.8176419109999999, 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00028140899999999999, 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:07.330715+0000", 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:07.330715+0000", 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.273 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:07.330717+0000", 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7400000000000001e-06 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:07.330727+0000", 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0543000000000001e-05 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:07.330755+0000", 2024-09-21T10:19:11.274 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.7310000000000001e-05 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:07.330768+0000", 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.332e-05 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:07.330886+0000", 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011828399999999999 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:07.330934+0000", 2024-09-21T10:19:11.275 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7769000000000003e-05 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:07.330997+0000", 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2442999999999998e-05 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "replica_scrub(pg: 1.5,from:0'0,to:0'0,epoch:303/296,start:1:a0000000::::0,end:MAX,chunky:1,deep:0,version:9,allow_preemption:1,priority=5)", 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:08.331365+0000", 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 2.8169923099999998, 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00033005399999999998, 2024-09-21T10:19:11.276 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:08.331365+0000", 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:08.331365+0000", 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.277 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:08.331366+0000", 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.593e-06 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:08.331377+0000", 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.062e-05 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:08.331414+0000", 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6733000000000002e-05 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.278 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:08.331430+0000", 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6161999999999999e-05 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:08.331619+0000", 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018908099999999999 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:08.331640+0000", 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0783000000000001e-05 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.279 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:08.331695+0000", 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5081999999999999e-05 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "replica_scrub(pg: 1.2,from:0'0,to:0'0,epoch:303/296,start:1:40000000::::0,end:MAX,chunky:1,deep:0,version:9,allow_preemption:1,priority=5)", 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:10.331559+0000", 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.81679814100000003, 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000286233, 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:11.280 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:10.331559+0000", 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:10.331559+0000", 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:10.331561+0000", 2024-09-21T10:19:11.281 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.584e-06 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:10.331571+0000", 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0156000000000001e-05 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:10.331601+0000", 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.9813000000000001e-05 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:11.282 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:10.331616+0000", 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5685999999999999e-05 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:10.331765+0000", 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000148987 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:10.331816+0000", 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1047000000000001e-05 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:11.283 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:10.331845+0000", 2024-09-21T10:19:11.284 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.8960000000000001e-05 2024-09-21T10:19:11.284 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.284 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.284 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.284 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:11.284 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:11.284 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:19:11.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:11.348 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:11.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:11.450 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:11.477 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:11.579 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:11.597 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:19:11.604 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:11.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:11.611 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[101299]: 1.2 scrub starts 2024-09-21T10:19:11.611 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[101299]: 1.2 scrub ok 2024-09-21T10:19:11.611 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[101299]: pgmap v902: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:11.611 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1878271797' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:11.611 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:11.612 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[105748]: 1.2 scrub starts 2024-09-21T10:19:11.612 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[105748]: 1.2 scrub ok 2024-09-21T10:19:11.612 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[105748]: pgmap v902: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:11.612 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1878271797' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:11.612 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:11 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:11.706 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:11.732 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:11.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:11.834 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:11.860 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:11.962 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:11.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:11.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:11 smithi045 ceph-mon[95976]: 1.2 scrub starts 2024-09-21T10:19:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:11 smithi045 ceph-mon[95976]: 1.2 scrub ok 2024-09-21T10:19:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:11 smithi045 ceph-mon[95976]: pgmap v902: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:11 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1878271797' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:11 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:12.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:12.114 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:12.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:12.215 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:12.237 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:12.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:12.339 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:12.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:12.463 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:12.485 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:12.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:12.587 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:12.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.611 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:12.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:12.712 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.735 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:12.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:12.836 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:12.862 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:12.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:12 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:19:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:12 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:12 smithi032 ceph-mon[101299]: osdmap e304: 8 total, 7 up, 8 in 2024-09-21T10:19:12.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:12 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10/87 objects degraded (11.494%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:12.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:12 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:19:12.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:12 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:12.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:12 smithi032 ceph-mon[105748]: osdmap e304: 8 total, 7 up, 8 in 2024-09-21T10:19:12.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:12 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10/87 objects degraded (11.494%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:12.964 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:12.991 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:13.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:12 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:19:13.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:12 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:13.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:12 smithi045 ceph-mon[95976]: osdmap e304: 8 total, 7 up, 8 in 2024-09-21T10:19:13.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:12 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10/87 objects degraded (11.494%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:13.093 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:13.120 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:13.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:13.221 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:13.248 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:13.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:13.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:13.375 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:13.470 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:19:13.477 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:13.539 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:13.557 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:13.561 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:13.641 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:13.662 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:13.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:13 smithi032 ceph-mon[101299]: 1.0 scrub starts 2024-09-21T10:19:13.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:13 smithi032 ceph-mon[101299]: 1.0 scrub ok 2024-09-21T10:19:13.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:13 smithi032 ceph-mon[101299]: pgmap v904: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:13.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:13 smithi032 ceph-mon[105748]: 1.0 scrub starts 2024-09-21T10:19:13.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:13 smithi032 ceph-mon[105748]: 1.0 scrub ok 2024-09-21T10:19:13.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:13 smithi032 ceph-mon[105748]: pgmap v904: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:13.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:13.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:13.889 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:13.912 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:13.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:14.013 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:13 smithi045 ceph-mon[95976]: 1.0 scrub starts 2024-09-21T10:19:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:13 smithi045 ceph-mon[95976]: 1.0 scrub ok 2024-09-21T10:19:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:13 smithi045 ceph-mon[95976]: pgmap v904: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:14.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:14.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:14.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:14.164 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:14.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:14.266 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:14.289 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:14.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:14.391 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:14.413 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:14.515 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:14.537 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:14.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:14.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:14.662 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:14.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:14.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:14 smithi032 ceph-mon[101299]: 1.1 deep-scrub starts 2024-09-21T10:19:14.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:14 smithi032 ceph-mon[101299]: 1.1 deep-scrub ok 2024-09-21T10:19:14.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:14 smithi032 ceph-mon[105748]: 1.1 deep-scrub starts 2024-09-21T10:19:14.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:14 smithi032 ceph-mon[105748]: 1.1 deep-scrub ok 2024-09-21T10:19:14.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:14.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:14.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:19:14.888 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:14.910 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:14.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:15.012 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:15.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:14 smithi045 ceph-mon[95976]: 1.1 deep-scrub starts 2024-09-21T10:19:15.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:14 smithi045 ceph-mon[95976]: 1.1 deep-scrub ok 2024-09-21T10:19:15.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:15.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:15.141 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:15.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:15.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:15.269 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:15.293 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:15.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:15.394 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:15.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.418 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:15.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:15.518 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:15.541 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:15.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:15.643 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.667 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:15.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:15.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:15 smithi032 ceph-mon[101299]: pgmap v905: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:15.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:15 smithi032 ceph-mon[105748]: pgmap v905: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:15.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:15.791 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:15.878 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:19:15.878 INFO:teuthology.orchestra.run.smithi032.stdout: "success": "" 2024-09-21T10:19:15.878 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:19:15.894 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:15.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:15.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:15.937 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:15 smithi045 ceph-mon[95976]: pgmap v905: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/87 objects degraded (11.494%); 5/87 objects misplaced (5.747%) 2024-09-21T10:19:16.023 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:16.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:16.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:16.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:16.170 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:16.272 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:16.294 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:16.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:16.395 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:16.417 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:16.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:16.519 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:16.546 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:16.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:16.647 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:16.649 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:16.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:16.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:16.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:16.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:16.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:16.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:16.922 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:16.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:16.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:16 smithi032 ceph-mon[105748]: 1.8 deep-scrub starts 2024-09-21T10:19:16.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:16 smithi032 ceph-mon[105748]: 1.8 deep-scrub ok 2024-09-21T10:19:16.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:16 smithi032 ceph-mon[101299]: 1.8 deep-scrub starts 2024-09-21T10:19:16.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:16 smithi032 ceph-mon[101299]: 1.8 deep-scrub ok 2024-09-21T10:19:17.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:16 smithi045 ceph-mon[95976]: 1.8 deep-scrub starts 2024-09-21T10:19:17.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:16 smithi045 ceph-mon[95976]: 1.8 deep-scrub ok 2024-09-21T10:19:17.047 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:17.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:17.118 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:17.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:17.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:17.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:17.277 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:17.302 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:17.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:17.429 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:17.532 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:17.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:17.660 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:17.684 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:17.786 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:17.813 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:17.909 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:17 smithi032 ceph-mon[101299]: pgmap v906: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/81 objects degraded (12.346%); 5/81 objects misplaced (6.173%) 2024-09-21T10:19:17.909 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10/81 objects degraded (12.346%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:17.910 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:17 smithi032 ceph-mon[105748]: pgmap v906: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/81 objects degraded (12.346%); 5/81 objects misplaced (6.173%) 2024-09-21T10:19:17.910 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10/81 objects degraded (12.346%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:17.914 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:17.941 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:17.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:18.010 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:19:18.010 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:18.027 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:19:18.027 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:19:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:17 smithi045 ceph-mon[95976]: pgmap v906: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/81 objects degraded (12.346%); 5/81 objects misplaced (6.173%) 2024-09-21T10:19:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10/81 objects degraded (12.346%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:18.042 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:18.043 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:19:18.044 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:18.060 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:19:18.060 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:18.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.065 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:18.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:18.077 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:19:18.078 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:18.078 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:19:18.094 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:19:18.095 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:18.111 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:19:18.111 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:18.167 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:18.193 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:18.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:18.295 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:18.318 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:18.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:18.421 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:18.448 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:18.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:18.550 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:18.573 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:18.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:18.674 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:18.701 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:18.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:18.802 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:18.824 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:18.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:18.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:18.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:18 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1101003632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:18.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:18 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1101003632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:18.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:18.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:18 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1101003632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:19.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:19.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:19.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:19.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:19.200 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:19.301 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:19.328 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:19.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:19.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:19.454 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:19.556 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:19.583 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:19.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:19.685 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:19.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:19 smithi032 ceph-mon[101299]: pgmap v907: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/81 objects degraded (12.346%); 5/81 objects misplaced (6.173%) 2024-09-21T10:19:19.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:19:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:19:19] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.51.0" 2024-09-21T10:19:19.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:19 smithi032 ceph-mon[105748]: pgmap v907: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/81 objects degraded (12.346%); 5/81 objects misplaced (6.173%) 2024-09-21T10:19:19.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:19.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:19.836 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:19.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:19.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:19 smithi045 ceph-mon[95976]: pgmap v907: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/81 objects degraded (12.346%); 5/81 objects misplaced (6.173%) 2024-09-21T10:19:20.062 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:20.084 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:20.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:20.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:20.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:20.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:20.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:20.333 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:20.334 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:20.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:20.458 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:20.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:20.583 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:20.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:20.685 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:20.712 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:20.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:20.814 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:20.837 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:20.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:20 smithi032 ceph-mon[105748]: 1.3 deep-scrub starts 2024-09-21T10:19:20.839 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:20 smithi032 ceph-mon[105748]: 1.3 deep-scrub ok 2024-09-21T10:19:20.839 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:20 smithi032 ceph-mon[101299]: 1.3 deep-scrub starts 2024-09-21T10:19:20.839 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:20 smithi032 ceph-mon[101299]: 1.3 deep-scrub ok 2024-09-21T10:19:20.940 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:20.967 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:20 smithi045 ceph-mon[95976]: 1.3 deep-scrub starts 2024-09-21T10:19:21.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:20 smithi045 ceph-mon[95976]: 1.3 deep-scrub ok 2024-09-21T10:19:21.069 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:21.093 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:21.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:21.195 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:21.216 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:21.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:21.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:21.345 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:21.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:21.447 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:21.469 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:21.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:21.521 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:19:21.522 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:19:21.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:21.593 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:21.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:21.634 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_3' created 2024-09-21T10:19:21.635 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-09-21T10:19:21.635 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:19:21.636 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [2, 3, 4, 5, 6, 7, 1, 0] 2024-09-21T10:19:21.636 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd out 4 2024-09-21T10:19:21.695 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[105748]: pgmap v908: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[105748]: 1.9 scrub starts 2024-09-21T10:19:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[105748]: 1.9 scrub ok 2024-09-21T10:19:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/770805127' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:19:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:19:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[101299]: pgmap v908: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[101299]: 1.9 scrub starts 2024-09-21T10:19:21.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[101299]: 1.9 scrub ok 2024-09-21T10:19:21.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/770805127' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:19:21.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:21 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:19:21.722 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:21.824 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:21.850 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:21.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:21.952 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:21.976 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:21.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:21 smithi045 ceph-mon[95976]: pgmap v908: 83 pgs: 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:21 smithi045 ceph-mon[95976]: 1.9 scrub starts 2024-09-21T10:19:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:21 smithi045 ceph-mon[95976]: 1.9 scrub ok 2024-09-21T10:19:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:21 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/770805127' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:19:22.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:21 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:19:22.078 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:22.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:22.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:22.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:22.232 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:22.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:22.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:22.357 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:22.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:22.459 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:22.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:22.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:22.513 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:19:22.513 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:19:22.588 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:22.613 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:22.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:22.715 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:22.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:22.840 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:22.863 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:22.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:22.944 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:22 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-21T10:19:22.944 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:22 smithi032 ceph-mon[101299]: osdmap e305: 8 total, 7 up, 8 in 2024-09-21T10:19:22.944 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:22.944 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:22 smithi032 ceph-mon[101299]: osdmap e306: 8 total, 7 up, 8 in 2024-09-21T10:19:22.944 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:22 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-21T10:19:22.944 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:22 smithi032 ceph-mon[105748]: osdmap e305: 8 total, 7 up, 8 in 2024-09-21T10:19:22.944 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:22.944 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:22 smithi032 ceph-mon[105748]: osdmap e306: 8 total, 7 up, 8 in 2024-09-21T10:19:22.965 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:22.966 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-09-21T10:19:22.990 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:22 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-21T10:19:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:22 smithi045 ceph-mon[95976]: osdmap e305: 8 total, 7 up, 8 in 2024-09-21T10:19:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:22 smithi045 ceph-mon[95976]: osdmap e306: 8 total, 7 up, 8 in 2024-09-21T10:19:23.092 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:23.117 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:23.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:23.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:23.241 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:23.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:23.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:23.365 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:23.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:23.467 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:23.470 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:23.491 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:23.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:23.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:23.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.617 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:23.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:23.718 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:23.740 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:23.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:23.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:23.864 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:23.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[101299]: pgmap v910: 99 pgs: 16 unknown, 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[101299]: 1.a scrub starts 2024-09-21T10:19:23.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[101299]: 1.a scrub ok 2024-09-21T10:19:23.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[101299]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:19:23.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[101299]: osdmap e307: 8 total, 7 up, 8 in 2024-09-21T10:19:23.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[105748]: pgmap v910: 99 pgs: 16 unknown, 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:23.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[105748]: 1.a scrub starts 2024-09-21T10:19:23.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[105748]: 1.a scrub ok 2024-09-21T10:19:23.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[105748]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:19:23.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:23 smithi032 ceph-mon[105748]: osdmap e307: 8 total, 7 up, 8 in 2024-09-21T10:19:23.966 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:23.993 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:23.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:23 smithi045 ceph-mon[95976]: pgmap v910: 99 pgs: 16 unknown, 12 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 54 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:23 smithi045 ceph-mon[95976]: 1.a scrub starts 2024-09-21T10:19:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:23 smithi045 ceph-mon[95976]: 1.a scrub ok 2024-09-21T10:19:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:23 smithi045 ceph-mon[95976]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:19:24.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:23 smithi045 ceph-mon[95976]: osdmap e307: 8 total, 7 up, 8 in 2024-09-21T10:19:24.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:24.122 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:24.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:24.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:24.246 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:24.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:24.348 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:24.372 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:24.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:24.474 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:24.501 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:24.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:24.514 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:19:24.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:24.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.624 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:24.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:24.725 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:24.747 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:24.849 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:24.871 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:24.973 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:24.995 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:25.096 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:25.123 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:25.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:25.225 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:25.251 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:25.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:25.353 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:25.380 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:25.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:25.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:25.505 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:25.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:25.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:25.633 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:25.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:25.735 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:25.759 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:25.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:25.861 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:25.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:25.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:25 smithi032 ceph-mon[101299]: pgmap v913: 99 pgs: 10 unknown, 12 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 56 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:25.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:25 smithi032 ceph-mon[105748]: pgmap v913: 99 pgs: 10 unknown, 12 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 56 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:25.985 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:26.012 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:26.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:26.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:25 smithi045 ceph-mon[95976]: pgmap v913: 99 pgs: 10 unknown, 12 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 56 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:26.090 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:26.111 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:19:26.113 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:26.176 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:26.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:26.278 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:26.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:26.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:26.406 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:26.427 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:26.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:26.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:26.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:26.557 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:26.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:26.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:26.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:26.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:26.803 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:26.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:26.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:26.926 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:26.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:27.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:27.050 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:27.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:27.152 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:27.178 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:27.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:27.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:26 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:19:26.960+0000 7f532c355640 -1 mon.a@0(leader).osd e307 definitely_dead 0 2024-09-21T10:19:27.280 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:27.306 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:27.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:27.408 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:27.430 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:27.435 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:27.537 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:27.560 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:27.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:27.656 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:19:27.662 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:27.666 INFO:teuthology.orchestra.run.smithi032.stderr:marked out osd.4. 2024-09-21T10:19:27.696 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:27.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:27.798 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:27.813 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[105748]: pgmap v914: 99 pgs: 2 unknown, 12 active+clean+remapped, 20 active+undersized, 6 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:27.813 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3519584724' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:27.814 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1073418572' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-21T10:19:27.814 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-21T10:19:27.814 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[105748]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-21T10:19:27.814 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[101299]: pgmap v914: 99 pgs: 2 unknown, 12 active+clean+remapped, 20 active+undersized, 6 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:27.814 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3519584724' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:27.814 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1073418572' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-21T10:19:27.814 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-21T10:19:27.814 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:27 smithi032 ceph-mon[101299]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-21T10:19:27.825 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:27.927 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:27.949 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:27.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:27 smithi045 ceph-mon[95976]: pgmap v914: 99 pgs: 2 unknown, 12 active+clean+remapped, 20 active+undersized, 6 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3519584724' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1073418572' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-21T10:19:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:27 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-21T10:19:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:27 smithi045 ceph-mon[95976]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-21T10:19:28.050 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:28.074 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:28.176 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:28.198 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:28.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:28.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:28.322 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:28.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:28.424 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:28.445 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:28.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:28.547 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:28.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.570 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:28.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:28.671 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:28.702 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:28.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:28.804 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:28.826 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:28.925 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:28.926 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3519584724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:28.926 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-21T10:19:28.926 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[105748]: osdmap e308: 8 total, 7 up, 7 in 2024-09-21T10:19:28.926 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:19:28.926 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:28.926 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3519584724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:28.926 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-21T10:19:28.926 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[101299]: osdmap e308: 8 total, 7 up, 7 in 2024-09-21T10:19:28.926 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:28 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:19:28.928 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:28.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:28 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:28 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3519584724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:28 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-21T10:19:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:28 smithi045 ceph-mon[95976]: osdmap e308: 8 total, 7 up, 7 in 2024-09-21T10:19:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:28 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:19:29.053 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:29.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:29.177 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:29.198 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:29.300 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:29.322 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:29.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:29.424 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:29.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:29.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:29.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:29.572 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:29.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:29.673 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:29.703 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:29.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:29.804 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:29.827 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:29.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:29.914 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:29 smithi032 ceph-mon[101299]: pgmap v916: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 6 active+undersized+degraded, 61 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:29.915 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:29 smithi032 ceph-mon[101299]: osdmap e309: 8 total, 7 up, 7 in 2024-09-21T10:19:29.915 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:29 smithi032 ceph-mon[105748]: pgmap v916: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 6 active+undersized+degraded, 61 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:29.915 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:29 smithi032 ceph-mon[105748]: osdmap e309: 8 total, 7 up, 7 in 2024-09-21T10:19:29.915 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:19:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:19:29] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.51.0" 2024-09-21T10:19:29.929 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:29.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:29.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:29 smithi045 ceph-mon[95976]: pgmap v916: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 6 active+undersized+degraded, 61 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 10/84 objects degraded (11.905%); 5/84 objects misplaced (5.952%) 2024-09-21T10:19:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:29 smithi045 ceph-mon[95976]: osdmap e309: 8 total, 7 up, 7 in 2024-09-21T10:19:30.052 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:30.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:30.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:30.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:30.198 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:30.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:30.300 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:30.322 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:30.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:30.424 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:30.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:30.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:30.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:30.570 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:30.672 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:30.698 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:30.800 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:30.811 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:30 smithi032 ceph-mon[105748]: osdmap e310: 8 total, 7 up, 7 in 2024-09-21T10:19:30.811 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:30 smithi032 ceph-mon[101299]: osdmap e310: 8 total, 7 up, 7 in 2024-09-21T10:19:30.822 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:30.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:30.924 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:30.946 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:30.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:30 smithi045 ceph-mon[95976]: osdmap e310: 8 total, 7 up, 7 in 2024-09-21T10:19:31.048 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:31.070 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:31.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:31.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:31.198 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:31.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:31.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:31.322 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:31.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:31.423 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:31.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.448 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:31.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:31.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:31.572 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:31.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:31.674 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:31.691 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:19:31.697 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:31.799 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:31.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.823 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:31.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:31.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:31 smithi032 ceph-mon[105748]: pgmap v919: 99 pgs: 1 active+undersized+remapped, 1 remapped+peering, 9 peering, 18 active+clean+remapped, 16 active+undersized, 4 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 0 B/s, 1 keys/s, 5 objects/s recovering 2024-09-21T10:19:31.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:31 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:19:31.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:31 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 8/84 objects degraded (9.524%), 4 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:31.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:31 smithi032 ceph-mon[105748]: osdmap e311: 8 total, 7 up, 7 in 2024-09-21T10:19:31.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:31 smithi032 ceph-mon[101299]: pgmap v919: 99 pgs: 1 active+undersized+remapped, 1 remapped+peering, 9 peering, 18 active+clean+remapped, 16 active+undersized, 4 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 0 B/s, 1 keys/s, 5 objects/s recovering 2024-09-21T10:19:31.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:31 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:19:31.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:31 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 8/84 objects degraded (9.524%), 4 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:31.826 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:31 smithi032 ceph-mon[101299]: osdmap e311: 8 total, 7 up, 7 in 2024-09-21T10:19:31.924 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:31.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:31.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:31.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:31 smithi045 ceph-mon[95976]: pgmap v919: 99 pgs: 1 active+undersized+remapped, 1 remapped+peering, 9 peering, 18 active+clean+remapped, 16 active+undersized, 4 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 0 B/s, 1 keys/s, 5 objects/s recovering 2024-09-21T10:19:31.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:31 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:19:31.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:31 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 8/84 objects degraded (9.524%), 4 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:31.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:31 smithi045 ceph-mon[95976]: osdmap e311: 8 total, 7 up, 7 in 2024-09-21T10:19:32.052 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:32.076 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:32.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:32.199 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:32.301 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:32.322 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:32.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:32.369 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:19:32.369 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:19:32.424 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:32.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:32.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:32.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:32.569 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:32.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:32.671 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:32.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:32.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:32.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:32.818 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:32.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:32.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[101299]: osdmap e312: 8 total, 7 up, 7 in 2024-09-21T10:19:32.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:32.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:32.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/293267441' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:19:32.844 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:19:32.845 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[105748]: osdmap e312: 8 total, 7 up, 7 in 2024-09-21T10:19:32.845 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:32.845 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:32.845 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/293267441' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:19:32.845 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:32 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:19:32.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:32.941 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:32.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:33.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:32 smithi045 ceph-mon[95976]: osdmap e312: 8 total, 7 up, 7 in 2024-09-21T10:19:33.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:32 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:33.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:32 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:33.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:32 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/293267441' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:19:33.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:32 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:19:33.043 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:33.065 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:33.167 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:33.190 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:33.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:33.244 INFO:teuthology.orchestra.run.smithi032.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-09-21T10:19:33.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:33.314 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:33.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:33.416 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:33.444 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:33.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:33.546 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:33.570 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:33.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:33.671 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:33.698 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:33.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:33.822 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:33.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:33.924 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:33.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.947 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:33.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[101299]: pgmap v922: 99 pgs: 1 active+undersized+remapped, 1 remapped+peering, 9 peering, 20 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 49 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 1.2 KiB/s, 2 keys/s, 8 objects/s recovering 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[101299]: osdmap e313: 8 total, 7 up, 7 in 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[105748]: pgmap v922: 99 pgs: 1 active+undersized+remapped, 1 remapped+peering, 9 peering, 20 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 49 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 1.2 KiB/s, 2 keys/s, 8 objects/s recovering 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[105748]: osdmap e313: 8 total, 7 up, 7 in 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:33 smithi045 ceph-mon[95976]: pgmap v922: 99 pgs: 1 active+undersized+remapped, 1 remapped+peering, 9 peering, 20 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 49 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 1.2 KiB/s, 2 keys/s, 8 objects/s recovering 2024-09-21T10:19:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:33 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:19:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:33 smithi045 ceph-mon[95976]: osdmap e313: 8 total, 7 up, 7 in 2024-09-21T10:19:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:34.038 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:19:34.048 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:34.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:34.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:34.201 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:34.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:34.232 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:19:34.232 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:34.251 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:19:34.251 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:19:34.272 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:19:34.272 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:34.294 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:19:34.294 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:34.311 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:19:34.311 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:34.312 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:19:34.328 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:19:34.329 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:34.330 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:34.345 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:19:34.345 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:34.353 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:34.455 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:34.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:34.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:34.494 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:34.583 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:34.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:34.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:34.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:34.734 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:34.835 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:34.858 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:34.931 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/238678455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:34.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/238678455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:34.959 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:34.981 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:34.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/238678455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:35.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:35.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:35.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:35.211 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:35.237 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:35.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:35.338 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:35.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:35.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:35.484 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:35.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:35.586 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:35.608 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:35.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:35.731 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:35.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:35.833 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:35.836 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 1, 0] out_osds: [4] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-09-21T10:19:35.836 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:19:35.836 INFO:tasks.thrashosds.thrasher:Adding osd 4 2024-09-21T10:19:35.836 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd in 4 2024-09-21T10:19:35.859 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:35 smithi032 ceph-mon[101299]: pgmap v924: 99 pgs: 1 active+undersized+remapped, 23 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 1.0 KiB/s, 2 keys/s, 7 objects/s recovering 2024-09-21T10:19:35.859 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:35 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-21T10:19:35.859 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:35 smithi032 ceph-mon[101299]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:19:35.859 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:35 smithi032 ceph-mon[105748]: pgmap v924: 99 pgs: 1 active+undersized+remapped, 23 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 1.0 KiB/s, 2 keys/s, 7 objects/s recovering 2024-09-21T10:19:35.859 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:35 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-21T10:19:35.859 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:35 smithi032 ceph-mon[105748]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:19:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.860 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:35.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:35.954 DEBUG:teuthology.orchestra.run.smithi032:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-09-21T10:19:35.956 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:19:35.956 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:19:35.961 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:35.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:35.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:36.007 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:hints = 1 2024-09-21T10:19:36.007 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:19:36.007 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object prefix: benchmark_data_smithi032_233595 2024-09-21T10:19:36.007 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:19:36.007 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:19:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:35 smithi045 ceph-mon[95976]: pgmap v924: 99 pgs: 1 active+undersized+remapped, 23 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 1.0 KiB/s, 2 keys/s, 7 objects/s recovering 2024-09-21T10:19:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:35 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-21T10:19:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:35 smithi045 ceph-mon[95976]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:19:36.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:36.131 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:36.232 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:36.274 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:36.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:36.376 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:36.403 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:36.505 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:36.532 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:36.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:36.634 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:36.661 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:36.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:36.761 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:19:36.761 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:19:36.763 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:36.790 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:36.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:36.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:36.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:37.008 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 1 16 3397 3381 211.3 211.312 0.00780424 0.00471702 2024-09-21T10:19:37.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:37.049 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:37.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:37.151 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:37.189 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:37.291 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:37.319 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:37.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:37.420 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:37.466 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:37.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:37.595 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:37.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:37.696 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:37.734 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:37.836 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:37.864 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:37.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:37.918 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:37.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:37 smithi032 ceph-mon[101299]: pgmap v925: 99 pgs: 23 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 14 B/s, 0 objects/s recovering 2024-09-21T10:19:37.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:37 smithi032 ceph-mon[105748]: pgmap v925: 99 pgs: 23 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 14 B/s, 0 objects/s recovering 2024-09-21T10:19:37.966 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:37.991 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:37.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:38.008 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 2 16 6532 6516 203.57 195.938 0.00431463 0.00490233 2024-09-21T10:19:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:37 smithi045 ceph-mon[95976]: pgmap v925: 99 pgs: 23 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 8/84 objects degraded (9.524%); 7/84 objects misplaced (8.333%); 14 B/s, 0 objects/s recovering 2024-09-21T10:19:38.094 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:38.137 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:38.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:38.239 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:38.282 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:38.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:38.384 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:38.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.410 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:38.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:38.511 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:38.519 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:38.572 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:38.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:38.674 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:38.701 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:38.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:38.762 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:19:38.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.827 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:38.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:38.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:38.954 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:38.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:39.008 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 3 16 8959 8943 186.271 151.688 0.00904858 0.00536149 2024-09-21T10:19:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:38 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 8/1152 objects degraded (0.694%), 4 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:39.056 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:39.099 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:39.201 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:38 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 8/1152 objects degraded (0.694%), 4 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:38 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 8/1152 objects degraded (0.694%), 4 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:39.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:39.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:39.363 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:39.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:39.465 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:39.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:39.605 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:39.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.636 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:39.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:39.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:39.751 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:19:39.781 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:39.883 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:39.923 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:39.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:39.930 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:19:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:19:39] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-09-21T10:19:39.930 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:39 smithi032 ceph-mon[101299]: pgmap v926: 99 pgs: 23 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 57 active+clean; 27 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2.9 MiB/s wr, 45 op/s; 8/1152 objects degraded (0.694%); 114/1152 objects misplaced (9.896%); 11 B/s, 0 objects/s recovering 2024-09-21T10:19:39.930 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/701197187' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:39.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:39 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:39.931 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:39 smithi032 ceph-mon[105748]: pgmap v926: 99 pgs: 23 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 57 active+clean; 27 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2.9 MiB/s wr, 45 op/s; 8/1152 objects degraded (0.694%); 114/1152 objects misplaced (9.896%); 11 B/s, 0 objects/s recovering 2024-09-21T10:19:39.931 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/701197187' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:39.931 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:39 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:40.009 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 4 16 11297 11281 176.175 146.125 0.0110233 0.0056676 2024-09-21T10:19:40.025 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:39 smithi045 ceph-mon[95976]: pgmap v926: 99 pgs: 23 active+clean+remapped, 15 active+undersized, 4 active+undersized+degraded, 57 active+clean; 27 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2.9 MiB/s wr, 45 op/s; 8/1152 objects degraded (0.694%); 114/1152 objects misplaced (9.896%); 11 B/s, 0 objects/s recovering 2024-09-21T10:19:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:39 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/701197187' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:39 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:40.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:40.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:40.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:40.216 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:40.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:40.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:40.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:40.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:40.449 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:40.488 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:40.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:40.590 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.631 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:40.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:40.732 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:40.755 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:40.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:40.856 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:40.883 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:40.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:40.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:41.006 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:41.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:41.009 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 5 16 13575 13559 169.415 142.375 0.00334261 0.0058532 2024-09-21T10:19:41.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:40 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:41.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:40 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:41.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:40 smithi045 ceph-mon[95976]: osdmap e314: 8 total, 7 up, 7 in 2024-09-21T10:19:41.108 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:41.146 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:41.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:41.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:40 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:41.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:40 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:41.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:40 smithi032 ceph-mon[101299]: osdmap e314: 8 total, 7 up, 7 in 2024-09-21T10:19:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:40 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:40 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:41.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:40 smithi032 ceph-mon[105748]: osdmap e314: 8 total, 7 up, 7 in 2024-09-21T10:19:41.248 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:41.279 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:41.381 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:41.408 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:41.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:41.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:41.535 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:41.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:41.637 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:41.665 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:41.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:41.766 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:41.803 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:41.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:41.905 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:41.927 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:41.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:42.009 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 6 16 15642 15626 162.71 129.188 0.00911405 0.00613898 2024-09-21T10:19:42.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:42.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:41 smithi045 ceph-mon[95976]: pgmap v928: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 523 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s; 3702/24957 objects degraded (14.834%); 1237/24957 objects misplaced (4.957%); 11 B/s, 0 objects/s recovering 2024-09-21T10:19:42.056 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:42.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:42.158 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:42.185 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:42.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:42.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:41 smithi032 ceph-mon[105748]: pgmap v928: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 523 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s; 3702/24957 objects degraded (14.834%); 1237/24957 objects misplaced (4.957%); 11 B/s, 0 objects/s recovering 2024-09-21T10:19:42.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:41 smithi032 ceph-mon[101299]: pgmap v928: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 523 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s; 3702/24957 objects degraded (14.834%); 1237/24957 objects misplaced (4.957%); 11 B/s, 0 objects/s recovering 2024-09-21T10:19:42.287 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:42.327 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:42.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:42.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:42.459 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:42.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:42.561 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:42.585 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:42.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:42.686 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:42.723 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:42.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:42.825 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:42.863 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:42.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:42.964 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:42.986 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:42.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:43.010 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 7 16 17616 17600 157.09 123.375 0.00159465 0.00630466 2024-09-21T10:19:43.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:43.121 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:43.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:43.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:43.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:43.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:43.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:43.401 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:43.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:43.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:43.529 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:43.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:43.631 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:43.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:43.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:43.757 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:43.780 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:43.881 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:43.908 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:43.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:44.010 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:44.010 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 8 16 19981 19965 155.929 147.812 0.00539242 0.00640704 2024-09-21T10:19:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:43 smithi045 ceph-mon[95976]: pgmap v929: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 676 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 3702/32292 objects degraded (11.464%); 2033/32292 objects misplaced (6.296%); 9 B/s, 0 objects/s recovering 2024-09-21T10:19:44.054 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:44.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:44.091 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:43 smithi032 ceph-mon[101299]: pgmap v929: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 676 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 3702/32292 objects degraded (11.464%); 2033/32292 objects misplaced (6.296%); 9 B/s, 0 objects/s recovering 2024-09-21T10:19:44.091 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:43 smithi032 ceph-mon[105748]: pgmap v929: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 676 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 3702/32292 objects degraded (11.464%); 2033/32292 objects misplaced (6.296%); 9 B/s, 0 objects/s recovering 2024-09-21T10:19:44.156 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:44.182 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:44.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:44.284 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:44.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.326 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:44.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:44.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:44.454 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:44.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:44.556 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:44.593 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:44.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:44.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:44.736 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:44.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:44.838 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:44.878 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:44.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:44.979 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:45.007 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:45.010 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 9 16 22357 22341 155.102 148.5 0.00955923 0.00644136 2024-09-21T10:19:45.109 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:45.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:45.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:44 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 5840/47124 objects degraded (12.393%), 10 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:44 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 5840/47124 objects degraded (12.393%), 10 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:45.250 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:44 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 5840/47124 objects degraded (12.393%), 10 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:45.289 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:45.391 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:45.433 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:45.535 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:45.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:45.677 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:45.704 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:45.806 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:45.866 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:45.968 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:46.010 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 10 16 24589 24573 153.534 139.5 0.011764 0.00650659 2024-09-21T10:19:46.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:46.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:46.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:46.144 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:46.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:46.186 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:45 smithi045 ceph-mon[95976]: pgmap v930: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 985 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 5840/47124 objects degraded (12.393%); 3408/47124 objects misplaced (7.232%) 2024-09-21T10:19:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:45 smithi032 ceph-mon[105748]: pgmap v930: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 985 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 5840/47124 objects degraded (12.393%); 3408/47124 objects misplaced (7.232%) 2024-09-21T10:19:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:45 smithi032 ceph-mon[101299]: pgmap v930: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 985 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 5840/47124 objects degraded (12.393%); 3408/47124 objects misplaced (7.232%) 2024-09-21T10:19:46.245 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:46.286 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:46.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:46.387 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:46.423 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:46.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:46.525 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:46.552 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:46.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:46.575 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:19:46.654 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:46.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.702 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:46.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:46.803 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:46.834 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:46.935 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:46.968 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:46.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:47.010 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 11 15 26627 26612 151.161 127.438 0.00655909 0.00660946 2024-09-21T10:19:47.069 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:47.132 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:47.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:47.233 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:47.277 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:47.379 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:47.405 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:47.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:47.507 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:47.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:47.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:47.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:47.676 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:47.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:47.778 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:47.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:47.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:47.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:47.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:48.010 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 12 16 28492 28476 148.272 116.5 0.00496293 0.00673874 2024-09-21T10:19:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:47 smithi045 ceph-mon[95976]: pgmap v931: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 1.2 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 7881/59547 objects degraded (13.235%); 3408/59547 objects misplaced (5.723%) 2024-09-21T10:19:48.050 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:48.075 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:48.154 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:47 smithi032 ceph-mon[105748]: pgmap v931: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 1.2 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 7881/59547 objects degraded (13.235%); 3408/59547 objects misplaced (5.723%) 2024-09-21T10:19:48.154 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:47 smithi032 ceph-mon[101299]: pgmap v931: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 1.2 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 7881/59547 objects degraded (13.235%); 3408/59547 objects misplaced (5.723%) 2024-09-21T10:19:48.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:48.199 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:48.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:48.300 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:48.342 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:48.443 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:48.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:48.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:48.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:48.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:48.759 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:48.861 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:48.899 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:48.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:49.000 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:49.010 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 13 16 30070 30054 144.453 98.625 0.00413867 0.00691735 2024-09-21T10:19:49.025 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:49.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:49.172 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:49.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:48 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:19:48.728+0000 7f532c355640 -1 mon.a@0(leader).osd e314 definitely_dead 0 2024-09-21T10:19:49.274 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:49.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:49.411 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:49.436 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:49.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:49.537 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:49.584 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:49.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:49.638 INFO:teuthology.orchestra.run.smithi032.stderr:marked in osd.4. 2024-09-21T10:19:49.686 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:49.708 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:49.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:49 smithi045 ceph-mon[95976]: pgmap v932: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 1.3 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s; 7881/65664 objects degraded (12.002%); 4092/65664 objects misplaced (6.232%) 2024-09-21T10:19:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:49.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:49 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2167040520' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-21T10:19:49.809 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:49.831 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:49.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:49.932 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:49.956 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:19:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:19:49] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-21T10:19:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:49 smithi032 ceph-mon[105748]: pgmap v932: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 1.3 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s; 7881/65664 objects degraded (12.002%); 4092/65664 objects misplaced (6.232%) 2024-09-21T10:19:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:49.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:49 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2167040520' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-21T10:19:49.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:49 smithi032 ceph-mon[101299]: pgmap v932: 99 pgs: 23 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 57 active+clean; 1.3 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s; 7881/65664 objects degraded (12.002%); 4092/65664 objects misplaced (6.232%) 2024-09-21T10:19:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:19:49.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:49.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:49 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2167040520' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-21T10:19:49.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:50.011 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 14 16 30993 30977 138.255 57.6875 0.0026035 0.00699649 2024-09-21T10:19:50.059 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:50.083 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:50.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:50.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:50.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:50.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:50.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:50.336 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:50.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:50.437 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:50.464 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:50.566 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:50.603 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:50.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:50.705 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:50.730 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:50.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:50.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:50.859 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:50.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:50 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2167040520' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-21T10:19:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:50 smithi032 ceph-mon[101299]: osdmap e315: 8 total, 7 up, 8 in 2024-09-21T10:19:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:50 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2167040520' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-21T10:19:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:50 smithi032 ceph-mon[105748]: osdmap e315: 8 total, 7 up, 8 in 2024-09-21T10:19:50.961 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:50.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:50.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:51.011 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 15 16 30993 30977 129.039 0 - 0.00699649 2024-09-21T10:19:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:50 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2167040520' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-21T10:19:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:50 smithi045 ceph-mon[95976]: osdmap e315: 8 total, 7 up, 8 in 2024-09-21T10:19:51.089 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:51.115 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:51.217 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:51.243 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:51.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:51.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:51.373 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:51.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:51.474 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:51.499 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:51.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:51.533 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:51.601 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:51.626 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:51.728 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:51.753 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:51.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:51.778 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:51 smithi032 ceph-mon[101299]: pgmap v934: 99 pgs: 10 remapped+peering, 20 peering, 13 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 37 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s; 11329/88407 objects degraded (12.815%); 5371/88407 objects misplaced (6.075%) 2024-09-21T10:19:51.778 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:51 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 8 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-21T10:19:51.779 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:51 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 11329/88407 objects degraded (12.815%), 10 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:51.779 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:51 smithi032 ceph-mon[101299]: osdmap e316: 8 total, 7 up, 8 in 2024-09-21T10:19:51.779 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:51 smithi032 ceph-mon[105748]: pgmap v934: 99 pgs: 10 remapped+peering, 20 peering, 13 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 37 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s; 11329/88407 objects degraded (12.815%); 5371/88407 objects misplaced (6.075%) 2024-09-21T10:19:51.779 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:51 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 8 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-21T10:19:51.779 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:51 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 11329/88407 objects degraded (12.815%), 10 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:51.779 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:51 smithi032 ceph-mon[105748]: osdmap e316: 8 total, 7 up, 8 in 2024-09-21T10:19:51.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:51.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:51.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:51.998 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:52.011 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 16 15 31692 31677 123.709 21.875 0.0114975 0.00807675 2024-09-21T10:19:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:51 smithi045 ceph-mon[95976]: pgmap v934: 99 pgs: 10 remapped+peering, 20 peering, 13 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 37 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s; 11329/88407 objects degraded (12.815%); 5371/88407 objects misplaced (6.075%) 2024-09-21T10:19:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:51 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 8 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-21T10:19:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:51 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 11329/88407 objects degraded (12.815%), 10 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:51 smithi045 ceph-mon[95976]: osdmap e316: 8 total, 7 up, 8 in 2024-09-21T10:19:52.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:52.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:52.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:52.176 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:52.278 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:52.306 INFO:tasks.thrashosds.thrasher:Added osd 4 2024-09-21T10:19:52.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.307 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:52.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:52.408 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:52.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:52.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:52.548 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:52.588 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:52.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:52.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:52.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.729 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:52.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:52.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:52.855 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:52.956 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:52.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:52 smithi032 ceph-mon[101299]: osdmap e317: 8 total, 7 up, 8 in 2024-09-21T10:19:52.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:52 smithi032 ceph-mon[105748]: osdmap e317: 8 total, 7 up, 8 in 2024-09-21T10:19:52.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:53.011 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 17 16 33433 33417 122.829 108.75 0.00854039 0.0081355 2024-09-21T10:19:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:52 smithi045 ceph-mon[95976]: osdmap e317: 8 total, 7 up, 8 in 2024-09-21T10:19:53.081 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:53.118 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:53.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:53.219 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:53.241 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:53.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:53.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:53.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.375 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:53.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:53.476 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:53.507 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:53.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:53.608 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:53.622 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246087 6.0 6:0f91fc0c:::benchmark_data_smithi032_171826_object116464:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:55.968070+0000", 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 357.60025475800001, 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7894905400000001, 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:19:53.623 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246087 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968070+0000", 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968070+0000", 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.624 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968071+0000", 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3540000000000001e-06 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968078+0000", 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4620000000000001e-06 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968097+0000", 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9459000000000001e-05 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.968107+0000", 2024-09-21T10:19:53.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7089999999999997e-06 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972504+0000", 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0043967340000000002 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op must wait for map", 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972510+0000", 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7660000000000001e-06 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.980092+0000", 2024-09-21T10:19:53.626 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0075817030000000004 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.980104+0000", 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1824000000000001e-05 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.561582+0000", 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.581478614 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.627 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562038+0000", 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00045512100000000001 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.753782+0000", 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1917440509999999 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.757560+0000", 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0037787430000000002 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246089 6.0 6:0ab96db1:::benchmark_data_smithi032_171826_object116466:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:55.972378+0000", 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 357.59594682099998, 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.785609827, 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:19:53.629 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246089 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972378+0000", 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.630 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972378+0000", 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972379+0000", 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.113e-06 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972384+0000", 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2429999999999996e-06 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.631 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972395+0000", 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1054e-05 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972403+0000", 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1810000000000001e-06 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972560+0000", 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000156522 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.632 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.972565+0000", 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8640000000000004e-06 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.980129+0000", 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.007564393 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:55.980137+0000", 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1990000000000006e-06 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.633 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562078+0000", 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.58194073199999996 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562089+0000", 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.08e-05 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.757913+0000", 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1958243159999999 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.634 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.757988+0000", 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4410000000000001e-05 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246161 6.0 6:0dcd9ebe:::benchmark_data_smithi032_171826_object116538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:56.067723+0000", 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 357.50060109899999, 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7490227869999999, 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:19:53.635 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246161 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067723+0000", 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067723+0000", 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.636 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067724+0000", 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.009e-06 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067729+0000", 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7210000000000002e-06 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067740+0000", 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0417e-05 2024-09-21T10:19:53.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067745+0000", 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4589999999999997e-06 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067798+0000", 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2703e-05 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.638 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.067807+0000", 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0499999999999997e-06 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562194+0000", 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.49438728999999998 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.562203+0000", 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6100000000000006e-06 2024-09-21T10:19:53.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.758010+0000", 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1958076310000001 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.759077+0000", 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0010662219999999999 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1,2", 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.759646+0000", 2024-09-21T10:19:53.640 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00056961100000000003 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798253+0000", 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.038607269 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798341+0000", 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7406999999999998e-05 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.641 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.805652+0000", 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0073109749999999999 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816575+0000", 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010922967 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816634+0000", 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8971000000000002e-05 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816746+0000", 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000112475 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246190 6.0 6:0bcfc80a:::benchmark_data_smithi032_171826_object116567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:56.680778+0000", 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 356.88754646799998, 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1366551, 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.643 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246190 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680778+0000", 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.644 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680778+0000", 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680779+0000", 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.085e-06 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680784+0000", 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8350000000000003e-06 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680797+0000", 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2509e-05 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680802+0000", 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3720000000000004e-06 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680871+0000", 2024-09-21T10:19:53.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9417000000000003e-05 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.680883+0000", 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1939999999999999e-05 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.789771+0000", 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1088872789999999 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.647 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.789889+0000", 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000118028 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1,2", 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.790005+0000", 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011647899999999999 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798294+0000", 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0082894289999999992 2024-09-21T10:19:53.648 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798376+0000", 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1923999999999997e-05 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.805752+0000", 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0073759380000000003 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.649 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.817252+0000", 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011500134 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.817307+0000", 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4462999999999997e-05 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.817433+0000", 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012626799999999999 2024-09-21T10:19:53.650 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.44852.0:246196 6.0 6:0d14539e:::benchmark_data_smithi032_171826_object116573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:13:56.690854+0000", 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 356.87747082499999, 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1304781020000001, 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.651 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.44852", 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/254151387", 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 246196 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690854+0000", 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690854+0000", 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690855+0000", 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2160000000000001e-06 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690859+0000", 2024-09-21T10:19:53.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6200000000000001e-06 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690870+0000", 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1367e-05 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690877+0000", 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6660000000000002e-06 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690937+0000", 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0338999999999999e-05 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:56.690948+0000", 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0713999999999999e-05 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798410+0000", 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1074626620000001 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.655 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798523+0000", 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0001131 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,1,2", 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.798656+0000", 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013238900000000001 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.805739+0000", 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0070834260000000003 2024-09-21T10:19:53.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816458+0000", 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010718459 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.816502+0000", 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4118e-05 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.821161+0000", 2024-09-21T10:19:53.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0046593590000000001 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.821218+0000", 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7184e-05 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:13:57.821332+0000", 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000113483 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.658 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2387 7.a 7:5446da43:::benchmark_data_smithi032_200324_object2386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.514140+0000", 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 309.05418439499999, 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2841524579999999, 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:19:53.659 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2387 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514140+0000", 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514140+0000", 2024-09-21T10:19:53.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514141+0000", 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3600000000000003e-07 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514164+0000", 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2730000000000001e-05 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514199+0000", 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5716000000000002e-05 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514207+0000", 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2810000000000003e-06 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514530+0000", 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00032313699999999998 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514676+0000", 2024-09-21T10:19:53.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014658499999999999 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.514823+0000", 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000146162 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.517811+0000", 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002988756 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.520195+0000", 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0023832739999999999 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.523380+0000", 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0031850680000000001 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.523389+0000", 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5000000000000005e-06 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521321+0000", 2024-09-21T10:19:53.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.99793176100000003 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521346+0000", 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5525e-05 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798199+0000", 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.276853062 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798293+0000", 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3165000000000007e-05 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2395 7.a 7:58b2188b:::benchmark_data_smithi032_200324_object2394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.516536+0000", 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 309.05178830900002, 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2821265199999998, 2024-09-21T10:19:53.666 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2395 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516536+0000", 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516536+0000", 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516537+0000", 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2799999999999995e-07 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516555+0000", 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8026e-05 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.668 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516565+0000", 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.7599999999999997e-06 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516572+0000", 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1740000000000003e-06 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516761+0000", 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018932399999999999 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.516886+0000", 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012498899999999999 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.517020+0000", 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013357500000000001 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.518943+0000", 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001922813 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.524409+0000", 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0054666100000000002 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.524420+0000", 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1077000000000001e-05 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521389+0000", 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.99696905700000005 2024-09-21T10:19:53.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521401+0000", 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1167e-05 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798572+0000", 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2771710330000001 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798663+0000", 2024-09-21T10:19:53.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1187000000000006e-05 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2437 7.a 7:5ed8f833:::benchmark_data_smithi032_200324_object2436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.546186+0000", 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 309.02213851300002, 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2556801950000001, 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:53.673 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2437 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546186+0000", 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546186+0000", 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546187+0000", 2024-09-21T10:19:53.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0299999999999998e-07 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546238+0000", 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1282999999999997e-05 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546262+0000", 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4000000000000001e-05 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546270+0000", 2024-09-21T10:19:53.675 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0649999999999999e-06 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546345+0000", 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5184000000000004e-05 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.546357+0000", 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1989e-05 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521431+0000", 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.97507342799999996 2024-09-21T10:19:53.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521439+0000", 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3159999999999997e-06 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798712+0000", 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2772729140000001 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.798852+0000", 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000139624 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799003+0000", 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015124999999999999 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.800437+0000", 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014343559999999999 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801083+0000", 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00064619600000000003 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801513+0000", 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00042902100000000002 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801585+0000", 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2937999999999999e-05 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801866+0000", 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000280828 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45206.0:2439 7.a 7:5a372019:::benchmark_data_smithi032_200324_object2438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:14:44.548988+0000", 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 309.01933666600002, 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2546032619999998, 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.680 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45206", 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 2439 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.548988+0000", 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.548988+0000", 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.681 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.548990+0000", 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1490000000000001e-06 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549047+0000", 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6978000000000002e-05 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549066+0000", 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8821000000000002e-05 2024-09-21T10:19:53.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549075+0000", 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4080000000000004e-06 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549139+0000", 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3781000000000005e-05 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:44.549147+0000", 2024-09-21T10:19:53.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3049999999999993e-06 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521467+0000", 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.97231946499999999 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:45.521475+0000", 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.678e-06 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799623+0000", 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2781484750000001 2024-09-21T10:19:53.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799792+0000", 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016894100000000001 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.799965+0000", 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017336700000000001 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801098+0000", 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001132463 2024-09-21T10:19:53.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.801924+0000", 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00082578700000000003 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.802129+0000", 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000205732 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.802360+0000", 2024-09-21T10:19:53.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00023111599999999999 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:14:46.803591+0000", 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0012307959999999999 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:13555 8.f 8:fabe3627:::benchmark_data_smithi032_233595_object13554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:40.965723+0000", 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 12.602601248999999, 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.131729755, 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.687 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 13555 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965723+0000", 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.688 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965723+0000", 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965724+0000", 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9999999999999997e-07 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965749+0000", 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.472e-05 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.689 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965758+0000", 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2070000000000001e-06 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965764+0000", 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4520000000000002e-06 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965846+0000", 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1934999999999994e-05 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.966254+0000", 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000407518 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.966392+0000", 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013849399999999999 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.968988+0000", 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0025960079999999999 2024-09-21T10:19:53.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.095970+0000", 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12698216200000001 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097347+0000", 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0013763709999999999 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.692 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097408+0000", 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1198999999999995e-05 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097453+0000", 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5089000000000001e-05 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:13556 8.0 8:02244fa0:::benchmark_data_smithi032_233595_object13555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:40.965962+0000", 2024-09-21T10:19:53.693 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 12.602362292, 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12745013999999999, 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 13556 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965962+0000", 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.694 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965962+0000", 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965963+0000", 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2999999999999999e-07 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.965986+0000", 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2646e-05 2024-09-21T10:19:53.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.966457+0000", 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00047160300000000002 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.966467+0000", 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4040000000000001e-06 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.966535+0000", 2024-09-21T10:19:53.696 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8242000000000002e-05 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.966698+0000", 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016343900000000001 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1", 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.966849+0000", 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015097900000000001 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.968470+0000", 2024-09-21T10:19:53.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001620233 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970776+0000", 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0023060469999999999 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970823+0000", 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7026e-05 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.093412+0000", 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.122589691 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:13562 8.f 8:fe23a469:::benchmark_data_smithi032_233595_object13561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:40.969963+0000", 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 12.598361549, 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.127641954, 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.699 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 13562 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.969963+0000", 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.969963+0000", 2024-09-21T10:19:53.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.969964+0000", 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.221e-06 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.969986+0000", 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1554000000000001e-05 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.701 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.969996+0000", 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0569e-05 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970003+0000", 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9410000000000001e-06 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970055+0000", 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1773e-05 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970192+0000", 2024-09-21T10:19:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013725099999999999 2024-09-21T10:19:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:19:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970328+0000", 2024-09-21T10:19:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013546599999999999 2024-09-21T10:19:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097463+0000", 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12713559599999999 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097496+0000", 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2214e-05 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097527+0000", 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1436999999999997e-05 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097575+0000", 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7688000000000001e-05 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097605+0000", 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0244e-05 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:13569 8.0 8:0b211dec:::benchmark_data_smithi032_233595_object13568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:40.970067+0000", 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 12.598258019999999, 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12858525600000001, 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 13569 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.706 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970067+0000", 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970067+0000", 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970067+0000", 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8999999999999997e-07 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970088+0000", 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0653e-05 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970095+0000", 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.729e-06 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970101+0000", 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7470000000000002e-06 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.093568+0000", 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12346705500000001 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.093725+0000", 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015640399999999999 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1", 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.093861+0000", 2024-09-21T10:19:53.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013596999999999999 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.098127+0000", 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0042660529999999997 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.098407+0000", 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00028002399999999997 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.098518+0000", 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011098 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.098652+0000", 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000134151 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:13559 8.7 e314/305 8:e6bcabdd:::benchmark_data_smithi032_233595_object13558:head v 314'861, mlcod=314'858)", 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:40.970149+0000", 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 12.598175815999999, 2024-09-21T10:19:53.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12804038300000001, 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970149+0000", 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970149+0000", 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970149+0000", 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2500000000000001e-07 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970177+0000", 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.8104999999999999e-05 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970185+0000", 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1329999999999999e-06 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970191+0000", 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.322e-06 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.093456+0000", 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.123264919 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.096556+0000", 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003100392 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.098147+0000", 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001591283 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.098189+0000", 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1903999999999997e-05 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.715 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:13563 8.7 e314/305 8:e7eb5b10:::benchmark_data_smithi032_233595_object13562:head v 314'862, mlcod=314'858)", 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:40.970456+0000", 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 12.597868113000001, 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12840515299999999, 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970456+0000", 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970456+0000", 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970457+0000", 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.58e-07 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970480+0000", 2024-09-21T10:19:53.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3561999999999999e-05 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970488+0000", 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4780000000000002e-06 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970495+0000", 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4209999999999998e-06 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.718 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097632+0000", 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.127136626 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097650+0000", 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7801000000000001e-05 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.098819+0000", 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0011692580000000001 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.098862+0000", 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2648999999999998e-05 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:13565 8.2 e314/309 8:44e279ec:::benchmark_data_smithi032_233595_object13564:head v 314'819, mlcod=314'816)", 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:40.970635+0000", 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 12.597689701, 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12676443400000001, 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.720 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970635+0000", 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970635+0000", 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970635+0000", 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9000000000000002e-07 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970658+0000", 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2765e-05 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970666+0000", 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1340000000000004e-06 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.970671+0000", 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0969999999999996e-06 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.094985+0000", 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12431384199999999 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.095009+0000", 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3603000000000001e-05 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097363+0000", 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0023540280000000002 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097399+0000", 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6575000000000001e-05 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.35586.0:13555 8.f e314/305 ondisk, result = 0)", 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:40.971150+0000", 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 12.597175049000001, 2024-09-21T10:19:53.724 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12631479500000001, 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.971150+0000", 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.971150+0000", 2024-09-21T10:19:53.725 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.726 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.971150+0000", 2024-09-21T10:19:53.726 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8299999999999997e-07 2024-09-21T10:19:53.726 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.726 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.726 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:40.971154+0000", 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1760000000000003e-06 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.093647+0000", 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.12249310099999999 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.093659+0000", 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1651000000000001e-05 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.727 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097325+0000", 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0036662309999999998 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097341+0000", 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5481e-05 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:41.097464+0000", 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012357199999999999 2024-09-21T10:19:53.728 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:30963 8.b 8:dfed482c:::benchmark_data_smithi032_233595_object30962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:49.540111+0000", 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 4.0282139829999997, 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0939000270000001, 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:19:53.729 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 30963 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.540111+0000", 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.540111+0000", 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.730 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.540111+0000", 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2399999999999998e-07 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.540137+0000", 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5386999999999999e-05 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.731 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.732 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:53 smithi032 ceph-mon[105748]: pgmap v937: 99 pgs: 1 activating+undersized+degraded+remapped, 10 remapped+peering, 20 peering, 12 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 13314/90276 objects degraded (14.748%); 5371/90276 objects misplaced (5.950%) 2024-09-21T10:19:53.732 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:53 smithi032 ceph-mon[105748]: osdmap e318: 8 total, 7 up, 8 in 2024-09-21T10:19:53.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:53 smithi032 ceph-mon[101299]: pgmap v937: 99 pgs: 1 activating+undersized+degraded+remapped, 10 remapped+peering, 20 peering, 12 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 13314/90276 objects degraded (14.748%); 5371/90276 objects misplaced (5.950%) 2024-09-21T10:19:53.732 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:53 smithi032 ceph-mon[101299]: osdmap e318: 8 total, 7 up, 8 in 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.540146+0000", 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0080000000000008e-06 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.540152+0000", 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3790000000000003e-06 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.733 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.541979+0000", 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001826753 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.542121+0000", 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014263200000000001 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.542268+0000", 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000147088 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.734 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.545726+0000", 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0034577570000000001 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.545739+0000", 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2901999999999999e-05 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:50.582480+0000", 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0367407319999999 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.735 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:50.582497+0000", 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6965000000000001e-05 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.627115+0000", 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0446179630000001 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for degraded object", 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.627629+0000", 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00051423300000000001 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.736 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.633906+0000", 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0062768700000000004 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.634011+0000", 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000104734 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:30989 8.b 8:df8b0a43:::benchmark_data_smithi032_233595_object30988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:49.578288+0000", 2024-09-21T10:19:53.737 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 3.9900362510000003, 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0595286970000002, 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 30989 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578288+0000", 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.738 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578288+0000", 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578290+0000", 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.451e-06 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578315+0000", 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5299999999999998e-05 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.739 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578331+0000", 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5549999999999999e-05 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578418+0000", 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7439000000000001e-05 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578482+0000", 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3811000000000003e-05 2024-09-21T10:19:53.740 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578492+0000", 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6360000000000006e-06 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:50.582731+0000", 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0042397649999999 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:50.582741+0000", 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3400000000000004e-06 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.741 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.628762+0000", 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0460213549999999 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.628937+0000", 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017509000000000001 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.629102+0000", 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016452200000000001 2024-09-21T10:19:53.742 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.632794+0000", 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0036920239999999999 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.634192+0000", 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001398676 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.743 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.636702+0000", 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0025098339999999998 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.636758+0000", 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5896999999999997e-05 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.637817+0000", 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001059007 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.744 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:30990 8.c 8:3fd0786d:::benchmark_data_smithi032_233595_object30989:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:19:49.578843+0000", 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 3.9894816479999999, 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9905835779999999, 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 30990 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:19:53.745 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578843+0000", 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578843+0000", 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578844+0000", 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2079999999999999e-06 2024-09-21T10:19:53.746 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578872+0000", 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.8296000000000001e-05 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578886+0000", 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3760999999999999e-05 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.747 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578894+0000", 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8739999999999995e-06 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578962+0000", 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8144000000000004e-05 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:49.578971+0000", 2024-09-21T10:19:53.748 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0499999999999997e-06 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:50.582973+0000", 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.004001691 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:50.582983+0000", 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0383999999999999e-05 2024-09-21T10:19:53.749 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.561142+0000", 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.97815855799999996 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.561354+0000", 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000212172 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:19:53.750 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.561544+0000", 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00019026199999999999 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.563346+0000", 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0018019329999999999 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.564656+0000", 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0013094669999999999 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:53.751 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.569239+0000", 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0045837050000000004 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.569304+0000", 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4678000000000007e-05 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:19:51.569427+0000", 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000122395 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.752 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:19:53.753 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:19:53.753 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:19:53.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:53.853 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:53.893 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:53.995 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:54.013 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 18 16 34830 34814 120.843 87.3125 0.0132338 0.00826914 2024-09-21T10:19:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:53 smithi045 ceph-mon[95976]: pgmap v937: 99 pgs: 1 activating+undersized+degraded+remapped, 10 remapped+peering, 20 peering, 12 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 13314/90276 objects degraded (14.748%); 5371/90276 objects misplaced (5.950%) 2024-09-21T10:19:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:53 smithi045 ceph-mon[95976]: osdmap e318: 8 total, 7 up, 8 in 2024-09-21T10:19:54.037 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:54.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:54.139 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:54.168 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:54.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:54.305 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:54.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:54.408 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:54.442 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:54.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:54.544 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:54.585 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:54.687 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:54.718 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:54.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:54.820 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:54.859 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:54.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:54.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:54 smithi032 ceph-mon[101299]: osdmap e319: 8 total, 7 up, 8 in 2024-09-21T10:19:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:54 smithi032 ceph-mon[105748]: osdmap e319: 8 total, 7 up, 8 in 2024-09-21T10:19:54.961 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:54.983 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:54.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:55.013 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 19 16 36586 36570 120.258 109.75 0.00392155 0.00828744 2024-09-21T10:19:55.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:54 smithi045 ceph-mon[95976]: osdmap e319: 8 total, 7 up, 8 in 2024-09-21T10:19:55.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:55.127 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:55.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:55.228 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:55.268 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:55.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:55.370 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:55.401 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:55.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:55.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:55.543 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:55.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:55.645 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:55.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:55.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:55.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:55.814 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:55.881 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:55 smithi032 ceph-mon[105748]: pgmap v940: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 10 remapped+peering, 20 peering, 11 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 37 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 16113/95664 objects degraded (16.843%); 5688/95664 objects misplaced (5.946%); 16 KiB/s, 0 objects/s recovering 2024-09-21T10:19:55.881 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:55 smithi032 ceph-mon[101299]: pgmap v940: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 10 remapped+peering, 20 peering, 11 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 37 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 16113/95664 objects degraded (16.843%); 5688/95664 objects misplaced (5.946%); 16 KiB/s, 0 objects/s recovering 2024-09-21T10:19:55.915 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:55.944 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:55.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:56.013 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:19:56.012451+0000 min lat: 0.00116053 max lat: 2.10857 avg lat: 0.00833705 2024-09-21T10:19:56.013 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:19:56.013 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 20 16 38387 38371 119.873 112.562 0.00999577 0.00833705 2024-09-21T10:19:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:55 smithi045 ceph-mon[95976]: pgmap v940: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 10 remapped+peering, 20 peering, 11 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 37 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 16113/95664 objects degraded (16.843%); 5688/95664 objects misplaced (5.946%); 16 KiB/s, 0 objects/s recovering 2024-09-21T10:19:56.046 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:56.081 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:56.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:56.183 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:56.207 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:56.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:56.309 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:56.331 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:56.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:56.457 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:56.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:56.559 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:56.597 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:56.698 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:56.740 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:56.842 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:56.874 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:56.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:56 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 17199/105261 objects degraded (16.339%), 12 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:56 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 9 pgs peering) 2024-09-21T10:19:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:56 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 17199/105261 objects degraded (16.339%), 12 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:56 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 9 pgs peering) 2024-09-21T10:19:56.976 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:57.008 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:57.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:57.013 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 21 16 39818 39802 118.424 89.4375 0.00900043 0.00843855 2024-09-21T10:19:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:56 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 17199/105261 objects degraded (16.339%), 12 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:19:57.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:56 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 9 pgs peering) 2024-09-21T10:19:57.109 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:57.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:57.251 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:57.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:57.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:57.307 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 1, 0, 4] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-09-21T10:19:57.307 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2024-09-21T10:19:57.307 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2024-09-21T10:19:57.308 DEBUG:teuthology.orchestra.run.smithi045:osd.5> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.5.service 2024-09-21T10:19:57.327 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl start ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.5 2024-09-21T10:19:57.385 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:57.423 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:57.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:57.526 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:57.548 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:57.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:57.650 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:57.687 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:57.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:57.738 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:57 smithi045 ceph-mon[95976]: pgmap v941: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 17199/105261 objects degraded (16.339%); 8150/105261 objects misplaced (7.743%); 4.9 MiB/s, 1 keys/s, 81 objects/s recovering 2024-09-21T10:19:57.738 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:57 smithi045 systemd[1]: Starting Ceph osd.5 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:19:57.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:57.827 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:57.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:57.929 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:57.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:57 smithi032 ceph-mon[105748]: pgmap v941: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 17199/105261 objects degraded (16.339%); 8150/105261 objects misplaced (7.743%); 4.9 MiB/s, 1 keys/s, 81 objects/s recovering 2024-09-21T10:19:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:57 smithi032 ceph-mon[101299]: pgmap v941: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 17199/105261 objects degraded (16.339%); 8150/105261 objects misplaced (7.743%); 4.9 MiB/s, 1 keys/s, 81 objects/s recovering 2024-09-21T10:19:57.964 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:57.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:58.006 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:19:58.014 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 22 16 41376 41360 117.466 97.375 0.00706983 0.00850662 2024-09-21T10:19:58.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:57 smithi045 podman[169172]: 2024-09-21 10:19:57.642755688 +0000 UTC m=+0.029434407 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:19:58.066 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:58.113 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:58.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:58.215 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:58.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:58.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:58.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:58.382 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:58.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:58.484 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:58.524 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:58.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:58.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:58.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:58.768 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:58.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:58.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:58.905 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:58.938 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:59.014 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 23 16 42973 42957 116.697 99.8125 0.00243384 0.00854527 2024-09-21T10:19:59.039 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:59.070 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:59.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:59.212 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:59.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:59.246 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:58 smithi045 podman[169172]: 2024-09-21 10:19:58.878475324 +0000 UTC m=+1.265154043 container create 042f17d9e1b01735da3c8b80774bca4db241fe55ddc870593c08b78cdd15b67e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T10:19:59.314 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:59.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:59.361 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:59.450 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:59.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:59.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:59.535 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 podman[169172]: 2024-09-21 10:19:59.373757729 +0000 UTC m=+1.760436468 container init 042f17d9e1b01735da3c8b80774bca4db241fe55ddc870593c08b78cdd15b67e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:19:59.535 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 podman[169172]: 2024-09-21 10:19:59.379588087 +0000 UTC m=+1.766266799 container start 042f17d9e1b01735da3c8b80774bca4db241fe55ddc870593c08b78cdd15b67e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-21T10:19:59.535 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 podman[169172]: 2024-09-21 10:19:59.482227851 +0000 UTC m=+1.868906581 container attach 042f17d9e1b01735da3c8b80774bca4db241fe55ddc870593c08b78cdd15b67e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux ) 2024-09-21T10:19:59.594 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:19:59.603 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:59.660 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:59.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:59.761 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:59.766 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:19:59 smithi032 ceph-mon[105748]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 17199/108291 objects degraded (15.882%); 8150/108291 objects misplaced (7.526%); 4.2 MiB/s, 1 keys/s, 69 objects/s recovering 2024-09-21T10:19:59.766 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:19:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:19:59] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-21T10:19:59.767 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:19:59 smithi032 ceph-mon[101299]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 17199/108291 objects degraded (15.882%); 8150/108291 objects misplaced (7.526%); 4.2 MiB/s, 1 keys/s, 69 objects/s recovering 2024-09-21T10:19:59.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:19:59.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:59.871 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:19:59 smithi045 ceph-mon[95976]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 17199/108291 objects degraded (15.882%); 8150/108291 objects misplaced (7.526%); 4.2 MiB/s, 1 keys/s, 69 objects/s recovering 2024-09-21T10:19:59.902 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:59.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.939 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:19:59.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:00.014 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 24 16 44473 44457 115.741 93.75 0.00314307 0.00860936 2024-09-21T10:20:00.040 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:00.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:00.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:00.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:00.199 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:00.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:00.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate[169288]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 bash[169172]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate[169288]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 bash[169172]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate[169288]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 bash[169172]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate[169288]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 bash[169172]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate[169288]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 bash[169172]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate[169288]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 bash[169172]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate[169288]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T10:20:00.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:19:59 smithi045 bash[169172]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T10:20:00.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:00.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.341 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:00.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:00.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:00.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:00.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:00.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:00.616 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:00.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:00.645 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:20:00.645 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:20:00.645 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:20:00.645 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:00.646 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:00.718 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:00.755 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:00.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:00.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:00.893 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:00.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 17199/108291 objects degraded (15.882%), 12 pgs degraded, 13 pgs undersized 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: osd.5 (root=default,host=smithi045) is down 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: [WRN] PG_DEGRADED: Degraded data redundancy: 17199/108291 objects degraded (15.882%), 12 pgs degraded, 13 pgs undersized 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 2.2 is stuck undersized for 2m, current state active+undersized+degraded, last acting [1,6] 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 3.2 is stuck undersized for 2m, current state active+undersized, last acting [3,6] 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 3.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 3.5 is stuck undersized for 2m, current state active+undersized, last acting [3,2] 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 3.c is stuck undersized for 2m, current state active+undersized, last acting [3,6] 2024-09-21T10:20:00.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 3.d is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,6] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 3.10 is stuck undersized for 2m, current state active+undersized+degraded, last acting [6,3] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 3.17 is stuck undersized for 2m, current state active+undersized, last acting [0,3] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 3.1f is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,6] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 4.3 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 4.12 is stuck undersized for 2m, current state active+undersized, last acting [6,2] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 4.15 is stuck undersized for 2m, current state active+undersized, last acting [7,3] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 4.1f is stuck undersized for 2m, current state active+undersized, last acting [6,1] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 8.0 is active+undersized+degraded, acting [3,1] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 8.3 is active+undersized+degraded, acting [1,6] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 8.8 is active+undersized+degraded, acting [3,7] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 8.9 is active+undersized+degraded, acting [6,3] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 8.a is active+undersized+degraded, acting [1,2] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 8.d is active+undersized+degraded, acting [2,6] 2024-09-21T10:20:00.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: pg 8.e is active+recovery_wait+undersized+degraded+remapped, acting [0,2] 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3164143196' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 17199/108291 objects degraded (15.882%), 12 pgs degraded, 13 pgs undersized 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: osd.5 (root=default,host=smithi045) is down 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: [WRN] PG_DEGRADED: Degraded data redundancy: 17199/108291 objects degraded (15.882%), 12 pgs degraded, 13 pgs undersized 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 2.2 is stuck undersized for 2m, current state active+undersized+degraded, last acting [1,6] 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 3.2 is stuck undersized for 2m, current state active+undersized, last acting [3,6] 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 3.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 3.5 is stuck undersized for 2m, current state active+undersized, last acting [3,2] 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 3.c is stuck undersized for 2m, current state active+undersized, last acting [3,6] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 3.d is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,6] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 3.10 is stuck undersized for 2m, current state active+undersized+degraded, last acting [6,3] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 3.17 is stuck undersized for 2m, current state active+undersized, last acting [0,3] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 3.1f is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,6] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 4.3 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 4.12 is stuck undersized for 2m, current state active+undersized, last acting [6,2] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 4.15 is stuck undersized for 2m, current state active+undersized, last acting [7,3] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 4.1f is stuck undersized for 2m, current state active+undersized, last acting [6,1] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 8.0 is active+undersized+degraded, acting [3,1] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 8.3 is active+undersized+degraded, acting [1,6] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 8.8 is active+undersized+degraded, acting [3,7] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 8.9 is active+undersized+degraded, acting [6,3] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 8.a is active+undersized+degraded, acting [1,2] 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-09-21T10:20:00.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 8.d is active+undersized+degraded, acting [2,6] 2024-09-21T10:20:00.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: pg 8.e is active+recovery_wait+undersized+degraded+remapped, acting [0,2] 2024-09-21T10:20:00.963 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:00 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3164143196' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:00.995 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:00.997 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 17199/108291 objects degraded (15.882%), 12 pgs degraded, 13 pgs undersized 2024-09-21T10:20:00.997 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: osd.5 (root=default,host=smithi045) is down 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: [WRN] PG_DEGRADED: Degraded data redundancy: 17199/108291 objects degraded (15.882%), 12 pgs degraded, 13 pgs undersized 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 2.2 is stuck undersized for 2m, current state active+undersized+degraded, last acting [1,6] 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 3.2 is stuck undersized for 2m, current state active+undersized, last acting [3,6] 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 3.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 3.5 is stuck undersized for 2m, current state active+undersized, last acting [3,2] 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 3.c is stuck undersized for 2m, current state active+undersized, last acting [3,6] 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 3.d is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,6] 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 3.10 is stuck undersized for 2m, current state active+undersized+degraded, last acting [6,3] 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 3.17 is stuck undersized for 2m, current state active+undersized, last acting [0,3] 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 3.1f is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,6] 2024-09-21T10:20:00.998 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 4.3 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 4.12 is stuck undersized for 2m, current state active+undersized, last acting [6,2] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 4.15 is stuck undersized for 2m, current state active+undersized, last acting [7,3] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 4.1f is stuck undersized for 2m, current state active+undersized, last acting [6,1] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 8.0 is active+undersized+degraded, acting [3,1] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 8.3 is active+undersized+degraded, acting [1,6] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 8.8 is active+undersized+degraded, acting [3,7] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 8.9 is active+undersized+degraded, acting [6,3] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 8.a is active+undersized+degraded, acting [1,2] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 8.d is active+undersized+degraded, acting [2,6] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: pg 8.e is active+recovery_wait+undersized+degraded+remapped, acting [0,2] 2024-09-21T10:20:00.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:00 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3164143196' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:01.015 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 25 16 45957 45941 114.816 92.75 0.00335024 0.00870446 2024-09-21T10:20:01.043 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:01.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:01.144 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:01.182 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:01.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:01.284 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:01.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:00 smithi045 podman[169172]: 2024-09-21 10:20:00.994138761 +0000 UTC m=+3.380817482 container died 042f17d9e1b01735da3c8b80774bca4db241fe55ddc870593c08b78cdd15b67e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:20:01.320 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:01.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:01.422 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:01.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:01.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:01.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:01.684 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:01.706 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:01.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:01.807 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:01.845 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:01.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:01.947 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:01.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:01 smithi032 ceph-mon[101299]: pgmap v943: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 20172/129234 objects degraded (15.609%); 8966/129234 objects misplaced (6.938%); 9.3 MiB/s, 1 keys/s, 150 objects/s recovering 2024-09-21T10:20:01.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:01 smithi032 ceph-mon[105748]: pgmap v943: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 20172/129234 objects degraded (15.609%); 8966/129234 objects misplaced (6.938%); 9.3 MiB/s, 1 keys/s, 150 objects/s recovering 2024-09-21T10:20:01.984 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:01.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:02.015 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 26 16 47340 47324 113.725 86.4375 0.00626841 0.00878831 2024-09-21T10:20:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:01 smithi045 ceph-mon[95976]: pgmap v943: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 20172/129234 objects degraded (15.609%); 8966/129234 objects misplaced (6.938%); 9.3 MiB/s, 1 keys/s, 150 objects/s recovering 2024-09-21T10:20:02.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:01 smithi045 podman[169495]: 2024-09-21 10:20:01.863231065 +0000 UTC m=+1.825974364 container remove 042f17d9e1b01735da3c8b80774bca4db241fe55ddc870593c08b78cdd15b67e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-activate, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T10:20:02.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.108 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:02.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:02.209 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:02.238 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:02.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:02.340 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:02.374 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:02.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:02.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:02.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:02.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:02.535 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:02 smithi045 podman[169726]: 2024-09-21 10:20:02.387747636 +0000 UTC m=+0.025645325 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:20:02.535 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:02 smithi045 podman[169726]: 2024-09-21 10:20:02.517493601 +0000 UTC m=+0.155391278 container create e525a7eff74a3049d475040435a339a72e12935258634244ea6d61120359e5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:20:02.613 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:02.636 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:02.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:02.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:02.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:02.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:02.879 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:02.890 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:20:02.912 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:02.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:02.944 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:02 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 20172/129234 objects degraded (15.609%), 12 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:02.944 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:02 smithi045 podman[169726]: 2024-09-21 10:20:02.796145544 +0000 UTC m=+0.434043243 container init e525a7eff74a3049d475040435a339a72e12935258634244ea6d61120359e5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T10:20:02.945 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:02 smithi045 podman[169726]: 2024-09-21 10:20:02.802169272 +0000 UTC m=+0.440066960 container start e525a7eff74a3049d475040435a339a72e12935258634244ea6d61120359e5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T10:20:02.945 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:02 smithi045 ceph-osd[169774]: -- 172.21.15.45:0/3363799976 <== mon.1 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5628bbdf31e0 con 0x5628bcb7e400 2024-09-21T10:20:02.945 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:02 smithi045 bash[169726]: e525a7eff74a3049d475040435a339a72e12935258634244ea6d61120359e5d1 2024-09-21T10:20:02.945 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:02 smithi045 systemd[1]: Started Ceph osd.5 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:20:02.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:02 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 20172/129234 objects degraded (15.609%), 12 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:02.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:02 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 20172/129234 objects degraded (15.609%), 12 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:03.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:03.015 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 27 16 48620 48604 112.475 80 0.0063528 0.00888519 2024-09-21T10:20:03.057 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:03.158 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:03.192 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:03.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:03.293 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:03.333 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:03.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:03.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:03.456 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:03.456 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:03.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:03.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:03.591 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:03.663 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:03 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:20:03.348+0000 7f235ab52640 -1 Falling back to public interface 2024-09-21T10:20:03.693 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:03.719 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:03.734 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:03.820 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:03.844 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:03.945 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:03 smithi032 ceph-mon[105748]: pgmap v944: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 20172/132078 objects degraded (15.273%); 8966/132078 objects misplaced (6.788%); 7.5 MiB/s, 1 keys/s, 121 objects/s recovering 2024-09-21T10:20:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:03 smithi032 ceph-mon[101299]: pgmap v944: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 20172/132078 objects degraded (15.273%); 8966/132078 objects misplaced (6.788%); 7.5 MiB/s, 1 keys/s, 121 objects/s recovering 2024-09-21T10:20:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:03.969 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:04.015 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 28 16 49412 49396 110.227 49.5 0.00256718 0.00897869 2024-09-21T10:20:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:03 smithi045 ceph-mon[95976]: pgmap v944: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 20172/132078 objects degraded (15.273%); 8966/132078 objects misplaced (6.788%); 7.5 MiB/s, 1 keys/s, 121 objects/s recovering 2024-09-21T10:20:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:04.072 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:04.094 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:04.171 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:20:04.196 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:04.218 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:04.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:04.320 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:04.343 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:04.445 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:04.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:04.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:04.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:04.595 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:04.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:04.696 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:04.726 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:04.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:04.828 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:04.850 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:04.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:04.900 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:04.952 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:04.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:04 smithi032 ceph-mon[105748]: osdmap e320: 8 total, 7 up, 8 in 2024-09-21T10:20:04.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:04 smithi032 ceph-mon[101299]: osdmap e320: 8 total, 7 up, 8 in 2024-09-21T10:20:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.989 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:04.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:05.015 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 29 16 49636 49620 106.909 14 0.00491944 0.00934745 2024-09-21T10:20:05.030 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:04 smithi045 ceph-mon[95976]: osdmap e320: 8 total, 7 up, 8 in 2024-09-21T10:20:05.090 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.131 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:05.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:05.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:05.265 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:05.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:05.367 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:05.393 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:05.495 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:05.528 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:05.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:05.630 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:05.684 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:05.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:05.786 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:05.832 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:05.933 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:05 smithi032 ceph-mon[105748]: pgmap v946: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 19541/139749 objects degraded (13.983%); 11241/139749 objects misplaced (8.044%); 7.9 MiB/s, 0 keys/s, 127 objects/s recovering 2024-09-21T10:20:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:05 smithi032 ceph-mon[105748]: osdmap e321: 8 total, 7 up, 8 in 2024-09-21T10:20:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:05 smithi032 ceph-mon[101299]: pgmap v946: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 19541/139749 objects degraded (13.983%); 11241/139749 objects misplaced (8.044%); 7.9 MiB/s, 0 keys/s, 127 objects/s recovering 2024-09-21T10:20:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:05 smithi032 ceph-mon[101299]: osdmap e321: 8 total, 7 up, 8 in 2024-09-21T10:20:05.962 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:06.016 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 30 16 51063 51047 106.318 89.1875 0.00975471 0.00939981 2024-09-21T10:20:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:05 smithi045 ceph-mon[95976]: pgmap v946: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 58 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 19541/139749 objects degraded (13.983%); 11241/139749 objects misplaced (8.044%); 7.9 MiB/s, 0 keys/s, 127 objects/s recovering 2024-09-21T10:20:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:05 smithi045 ceph-mon[95976]: osdmap e321: 8 total, 7 up, 8 in 2024-09-21T10:20:06.064 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:06.093 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:06.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:06.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:06.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:06.246 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:20:06.246 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:20:06.329 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:06.372 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:06.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:06.474 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:06.514 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:06.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:06.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:06.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:06.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:06.758 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:06.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:06.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:06.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:06.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:06.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:07.016 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 31 16 52308 52292 105.399 77.8125 0.0140059 0.00948166 2024-09-21T10:20:07.053 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:07.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:07.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:07.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:07.216 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:07.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:07.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:07.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:07.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:07.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.479 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:07.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:07.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:07.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:07.703 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:20:07.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:07.740 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:07.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:07.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:07.881 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:07.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:07 smithi032 ceph-mon[101299]: pgmap v948: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s; 20531/147591 objects degraded (13.911%); 11161/147591 objects misplaced (7.562%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:20:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:07 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 20531/147591 objects degraded (13.911%), 11 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:07 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/246705094' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:07 smithi032 ceph-mon[105748]: pgmap v948: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s; 20531/147591 objects degraded (13.911%); 11161/147591 objects misplaced (7.562%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:20:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:07 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 20531/147591 objects degraded (13.911%), 11 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:07 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/246705094' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:07.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:08.011 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:07 smithi045 ceph-mon[95976]: pgmap v948: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s; 20531/147591 objects degraded (13.911%); 11161/147591 objects misplaced (7.562%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:20:08.012 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:07 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 20531/147591 objects degraded (13.911%), 11 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:08.012 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:07 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/246705094' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:08.012 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:08.016 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 32 16 53377 53361 104.192 66.8125 0.0160917 0.00959193 2024-09-21T10:20:08.029 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:08.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:08.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:08.155 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi045.stderr:no valid command found; 10 closest matches: 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi045.stderr:0 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi045.stderr:1 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi045.stderr:2 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi045.stderr:abort 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi045.stderr:assert 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi045.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi045.stderr:bluefs files list 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi045.stderr:bluefs stats 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi045.stderr:bluestore allocator dump block 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi045.stderr:bluestore allocator fragmentation block 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi045.stderr:admin_socket: invalid command 2024-09-21T10:20:08.247 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:20:08.257 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:08.284 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:08.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:08.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:08.444 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:08.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:08.546 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:08.569 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:08.670 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:08.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:08.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:08.796 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:08.822 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:08.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:08.924 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:08.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:08 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:08.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:08 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:08.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:08 smithi032 ceph-mon[101299]: osdmap e322: 8 total, 7 up, 8 in 2024-09-21T10:20:08.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:08 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:08.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:08 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:08.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:08 smithi032 ceph-mon[105748]: osdmap e322: 8 total, 7 up, 8 in 2024-09-21T10:20:08.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:09.016 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 33 16 54194 54178 102.582 51.0625 0.0143961 0.00974352 2024-09-21T10:20:09.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:08 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:09.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:08 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:09.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:08 smithi045 ceph-mon[95976]: osdmap e322: 8 total, 7 up, 8 in 2024-09-21T10:20:09.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:09.106 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:09.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:09.136 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:20:09.136 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2024-09-21T10:20:09.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:09.243 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:09.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:09.369 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:09.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:09.470 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:09.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:09.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:09.642 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:09.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:09.784 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:09.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:09.886 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:09.923 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:09.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:09.933 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:09 smithi045 ceph-mon[95976]: pgmap v950: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 20531/149493 objects degraded (13.734%); 11161/149493 objects misplaced (7.466%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:20:09.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:09 smithi032 ceph-mon[105748]: pgmap v950: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 20531/149493 objects degraded (13.734%); 11161/149493 objects misplaced (7.466%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:20:09.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:09 smithi032 ceph-mon[101299]: pgmap v950: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 20531/149493 objects degraded (13.734%); 11161/149493 objects misplaced (7.466%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:20:09.960 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:20:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:20:09] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.51.0" 2024-09-21T10:20:10.018 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 34 16 55473 55457 101.91 79.9375 0.0148988 0.00980412 2024-09-21T10:20:10.025 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:10.078 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:10.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:10.180 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:10.224 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:10.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:10.266 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:10.266 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:10.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.366 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:10.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:10.467 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:10.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:10.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:10.605 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:10.663 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:10.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:10.718 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:10.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:10.785 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:10 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:20:10.586+0000 7f235ab52640 -1 osd.5 280 log_to_monitors true 2024-09-21T10:20:10.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:10 smithi045 ceph-mon[95976]: from='osd.5 [v2:172.21.15.45:6808/3383231394,v1:172.21.15.45:6809/3383231394]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:20:10.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:10 smithi045 ceph-mon[95976]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:20:10.804 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:10.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:10.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:10.930 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:10.935 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:10 smithi032 ceph-mon[105748]: from='osd.5 [v2:172.21.15.45:6808/3383231394,v1:172.21.15.45:6809/3383231394]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:20:10.935 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:10 smithi032 ceph-mon[105748]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:20:10.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:10 smithi032 ceph-mon[101299]: from='osd.5 [v2:172.21.15.45:6808/3383231394,v1:172.21.15.45:6809/3383231394]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:20:10.935 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:10 smithi032 ceph-mon[101299]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:20:10.937 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:20:11.018 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 35 16 56813 56797 101.391 83.75 0.0141298 0.00985736 2024-09-21T10:20:11.033 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:11.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.071 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:11.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:11.171 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:11.218 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:11.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:11.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:20:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:20:11.110+0000 7f234e8f4640 -1 osd.5 323 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:20:11.287 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:11.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:11.363 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:11.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:11.464 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:11.494 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:11.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:11.595 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:11.647 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:11.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:11.748 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:11.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.812 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:11.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:11.818 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[101299]: pgmap v951: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 22486/163509 objects degraded (13.752%); 11779/163509 objects misplaced (7.204%); 11 MiB/s, 178 objects/s recovering 2024-09-21T10:20:11.818 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[101299]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T10:20:11.818 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[101299]: from='osd.5 [v2:172.21.15.45:6808/3383231394,v1:172.21.15.45:6809/3383231394]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-21T10:20:11.818 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[101299]: osdmap e323: 8 total, 7 up, 8 in 2024-09-21T10:20:11.818 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[101299]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-21T10:20:11.818 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[101299]: from='osd.5 ' entity='osd.5' 2024-09-21T10:20:11.818 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:11.818 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:11.818 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[105748]: pgmap v951: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 22486/163509 objects degraded (13.752%); 11779/163509 objects misplaced (7.204%); 11 MiB/s, 178 objects/s recovering 2024-09-21T10:20:11.819 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[105748]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T10:20:11.819 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[105748]: from='osd.5 [v2:172.21.15.45:6808/3383231394,v1:172.21.15.45:6809/3383231394]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-21T10:20:11.819 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[105748]: osdmap e323: 8 total, 7 up, 8 in 2024-09-21T10:20:11.819 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[105748]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-21T10:20:11.819 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[105748]: from='osd.5 ' entity='osd.5' 2024-09-21T10:20:11.819 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:11.819 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:11 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:11.913 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:11.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:11.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:12.019 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 36 16 58125 58109 100.851 82 0.00757993 0.00990943 2024-09-21T10:20:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:11 smithi045 ceph-mon[95976]: pgmap v951: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 22486/163509 objects degraded (13.752%); 11779/163509 objects misplaced (7.204%); 11 MiB/s, 178 objects/s recovering 2024-09-21T10:20:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:11 smithi045 ceph-mon[95976]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T10:20:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:11 smithi045 ceph-mon[95976]: from='osd.5 [v2:172.21.15.45:6808/3383231394,v1:172.21.15.45:6809/3383231394]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-21T10:20:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:11 smithi045 ceph-mon[95976]: osdmap e323: 8 total, 7 up, 8 in 2024-09-21T10:20:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:11 smithi045 ceph-mon[95976]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-21T10:20:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:11 smithi045 ceph-mon[95976]: from='osd.5 ' entity='osd.5' 2024-09-21T10:20:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:11 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:11 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:12.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:12.104 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:12.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:12.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:12.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:12.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:12.353 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:12.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:12.455 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:12.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:12.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:12.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:12.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:12.650 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:20:12.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:12.728 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:12.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:12 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 22486/163509 objects degraded (13.752%), 11 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:12.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:12 smithi045 ceph-mon[95976]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:20:12.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:12 smithi045 ceph-mon[95976]: osd.5 [v2:172.21.15.45:6808/3383231394,v1:172.21.15.45:6809/3383231394] boot 2024-09-21T10:20:12.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:12 smithi045 ceph-mon[95976]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T10:20:12.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T10:20:12.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:20:12.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:20:12.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:12.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:12.851 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:12.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:12.911 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 22486/163509 objects degraded (13.752%), 11 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:12.911 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[105748]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:20:12.911 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[105748]: osd.5 [v2:172.21.15.45:6808/3383231394,v1:172.21.15.45:6809/3383231394] boot 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[105748]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 22486/163509 objects degraded (13.752%), 11 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[101299]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[101299]: osd.5 [v2:172.21.15.45:6808/3383231394,v1:172.21.15.45:6809/3383231394] boot 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[101299]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:20:12.912 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:12.954 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:12.976 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:13.019 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 37 16 58215 58199 98.2783 5.625 0.00247974 0.00991594 2024-09-21T10:20:13.078 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:13.091 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:13.091 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:13.107 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:13.209 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:13.233 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:13.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:13.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:20:13.414 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:20:13.415 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:20:13.415 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:13.415 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:13.415 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:13.452 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:13.460 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:13.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:13.507 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:20:13.507 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:13.527 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:20:13.528 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:13.547 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:20:13.547 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:13.565 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:20:13.565 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:13.582 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:20:13.582 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:13.583 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:13.608 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:13.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:13.709 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:13.731 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:13.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:13.833 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:13.856 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:13.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:13.980 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:13.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:14.019 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 38 16 58215 58199 95.6926 0 - 0.00991594 2024-09-21T10:20:14.081 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:14.104 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:14.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:14.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:20:14.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:14.236 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:14.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:14.337 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:14.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:14.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:14 smithi032 ceph-mon[101299]: pgmap v954: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 22486/165681 objects degraded (13.572%); 11779/165681 objects misplaced (7.109%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T10:20:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:14 smithi032 ceph-mon[101299]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T10:20:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:14 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3720612991' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:14 smithi032 ceph-mon[105748]: pgmap v954: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 22486/165681 objects degraded (13.572%); 11779/165681 objects misplaced (7.109%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T10:20:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:14 smithi032 ceph-mon[105748]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T10:20:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:14 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3720612991' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:14.462 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:14.500 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:14 smithi045 ceph-mon[95976]: pgmap v954: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+clean+remapped, 9 active+undersized, 10 active+undersized+degraded, 59 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 22486/165681 objects degraded (13.572%); 11779/165681 objects misplaced (7.109%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T10:20:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:14 smithi045 ceph-mon[95976]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T10:20:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:14 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3720612991' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:14.602 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:14.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:14.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:14.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:14.774 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:14.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:14.876 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:14.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.914 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:14.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:15.015 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:15.019 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 39 16 58734 58718 94.071 16.2188 0.0370973 0.0106245 2024-09-21T10:20:15.053 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:15.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:15.155 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:15.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:20:15.187 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:15.288 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:15.325 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:15.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:15.371 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:15.427 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:15.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:15 smithi032 ceph-mon[105748]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T10:20:15.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:15 smithi032 ceph-mon[101299]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T10:20:15.458 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:15.525 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:15 smithi045 ceph-mon[95976]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T10:20:15.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:15.599 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:15.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:15.700 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:15.742 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:15.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:15.844 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:15.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:15.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:15.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:16.019 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:20:16.018256+0000 min lat: 0.00116053 max lat: 2.48922 avg lat: 0.0107244 2024-09-21T10:20:16.019 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:20:16.019 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 40 16 59673 59657 93.1864 58.6875 0.0199466 0.0107244 2024-09-21T10:20:16.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:16.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:16.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:16.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:16.189 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:16 smithi032 ceph-mon[105748]: pgmap v957: 99 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+clean+remapped, 8 active+undersized, 7 active+undersized+degraded, 60 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 37 MiB/s wr, 585 op/s; 15981/170532 objects degraded (9.371%); 7713/170532 objects misplaced (4.523%); 9.6 MiB/s, 153 objects/s recovering 2024-09-21T10:20:16.190 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:16 smithi032 ceph-mon[101299]: pgmap v957: 99 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+clean+remapped, 8 active+undersized, 7 active+undersized+degraded, 60 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 37 MiB/s wr, 585 op/s; 15981/170532 objects degraded (9.371%); 7713/170532 objects misplaced (4.523%); 9.6 MiB/s, 153 objects/s recovering 2024-09-21T10:20:16.207 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:16.257 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:16.302 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:16.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:16.404 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.442 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:16.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:16 smithi045 ceph-mon[95976]: pgmap v957: 99 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+clean+remapped, 8 active+undersized, 7 active+undersized+degraded, 60 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 37 MiB/s wr, 585 op/s; 15981/170532 objects degraded (9.371%); 7713/170532 objects misplaced (4.523%); 9.6 MiB/s, 153 objects/s recovering 2024-09-21T10:20:16.543 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:16.582 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:16.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:16.684 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:16.720 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:16.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:16.822 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:16.857 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:16.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:16.959 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:16.997 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:16.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:17.019 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 41 16 60691 60675 92.4654 63.625 0.00989882 0.0108083 2024-09-21T10:20:17.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:17.143 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:17.245 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:17.290 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:17.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:17.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:17.424 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:17.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 15981/170532 objects degraded (9.371%), 8 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:17.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 15981/170532 objects degraded (9.371%), 8 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:17.471 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 15981/170532 objects degraded (9.371%), 8 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2024-09-21T10:20:17.526 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:17.563 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:17.664 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:17.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.701 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:17.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:17.802 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:17.845 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:17.946 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:17.973 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:17.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:18.019 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 42 16 61542 61526 91.5303 53.1875 0.0121401 0.0109195 2024-09-21T10:20:18.076 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:18.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.119 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:18.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:18.220 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.261 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:18.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:18.298 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:18 smithi045 ceph-mon[95976]: pgmap v958: 99 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 3 peering, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 23713/175233 objects degraded (13.532%); 793/175233 objects misplaced (0.453%); 6.7 MiB/s, 108 objects/s recovering 2024-09-21T10:20:18.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:18.400 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:18 smithi032 ceph-mon[101299]: pgmap v958: 99 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 3 peering, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 23713/175233 objects degraded (13.532%); 793/175233 objects misplaced (0.453%); 6.7 MiB/s, 108 objects/s recovering 2024-09-21T10:20:18.400 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:18 smithi032 ceph-mon[105748]: pgmap v958: 99 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 3 peering, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 23713/175233 objects degraded (13.532%); 793/175233 objects misplaced (0.453%); 6.7 MiB/s, 108 objects/s recovering 2024-09-21T10:20:18.401 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:18.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:18.546 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:18.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:18.551 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:61988 8.0 e326/325 8:0d88144f:::benchmark_data_smithi032_233595_object61987:head v 326'3912, mlcod=0'0)", 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:18.532140+0000", 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.016085552999999999, 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.016126094000000001, 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:18.552 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532140+0000", 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532140+0000", 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532139+0000", 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.553 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532182+0000", 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2716999999999998e-05 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532197+0000", 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5047e-05 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532204+0000", 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0049999999999998e-06 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.554 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.546102+0000", 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013898491000000001 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.546126+0000", 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3441000000000001e-05 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:61989 8.8 e326/325 8:1d515b16:::benchmark_data_smithi032_233595_object61988:head v 326'3914, mlcod=313'111)", 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:18.532211+0000", 2024-09-21T10:20:18.555 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.016014390999999999, 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.016122286, 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532211+0000", 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532211+0000", 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.556 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532211+0000", 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532243+0000", 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2288e-05 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532251+0000", 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8450000000000003e-06 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.557 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.532256+0000", 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2229999999999998e-06 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.541091+0000", 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0088347719999999994 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.541125+0000", 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4285999999999997e-05 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.558 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:61994 8.a e326/325 8:5219d7d0:::benchmark_data_smithi032_233595_object61993:head v 326'3736, mlcod=0'0)", 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:18.536752+0000", 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.011473445000000001, 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011654981999999999, 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.536752+0000", 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.559 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.536752+0000", 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.536751+0000", 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.536808+0000", 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6606e-05 2024-09-21T10:20:18.560 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.536822+0000", 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4535999999999999e-05 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.536830+0000", 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1720000000000007e-06 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.536983+0000", 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015265600000000001 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.561 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.537043+0000", 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9933999999999999e-05 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:62002 8.9 e326/325 8:9c46d187:::benchmark_data_smithi032_233595_object62001:head v 326'3904, mlcod=0'0)", 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:18.542293+0000", 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0059331990000000001, 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0061719260000000003, 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:20:18.562 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.542293+0000", 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.542293+0000", 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.542291+0000", 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:18.563 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.542321+0000", 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9957000000000001e-05 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.542340+0000", 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8664e-05 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.542347+0000", 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.695e-06 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.564 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.542830+0000", 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00048248600000000002 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.542858+0000", 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7761000000000001e-05 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:18.565 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:61997 8.3 e326/325 8:c6b24f3f:::benchmark_data_smithi032_233595_object61996:head v 326'3848, mlcod=0'0)", 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:18.547541+0000", 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00068430200000000002, 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00098024099999999997, 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547541+0000", 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.566 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547541+0000", 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547541+0000", 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547577+0000", 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6386e-05 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.567 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547589+0000", 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1926000000000001e-05 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547596+0000", 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0809999999999996e-06 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547705+0000", 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000108795 2024-09-21T10:20:18.568 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547729+0000", 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4275000000000001e-05 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:62001 8.2 e326/325)", 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:18.547988+0000", 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00023792099999999999, 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00058574500000000004, 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:18.569 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547988+0000", 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547988+0000", 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.570 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.547987+0000", 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548013+0000", 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.6265999999999999e-05 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548028+0000", 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4630000000000001e-05 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.571 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548035+0000", 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8789999999999997e-06 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:61999 8.4 e326/325 8:291fbcaa:::benchmark_data_smithi032_233595_object61998:head v 326'3868, mlcod=0'0)", 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:18.548047+0000", 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00017872499999999999, 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00056758199999999996, 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.572 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548047+0000", 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548047+0000", 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548047+0000", 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.573 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548071+0000", 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4179e-05 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548077+0000", 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.392e-06 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548082+0000", 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5059999999999997e-06 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.574 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548148+0000", 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5993000000000003e-05 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:18.548169+0000", 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1229999999999998e-05 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 7 2024-09-21T10:20:18.575 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:20:18.648 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:18.686 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:18.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:20:18.788 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:18.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.883 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:18.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:18.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:19.020 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 43 16 62360 62344 90.589 51.125 0.0133435 0.011032 2024-09-21T10:20:19.030 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:19.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:19.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:19.139 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:19.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:19.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:19.263 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:19.299 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:19.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:19.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:19.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.441 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:19.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:19.536 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[101299]: pgmap v959: 99 pgs: 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 21728/176232 objects degraded (12.329%); 2625/176232 objects misplaced (1.490%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:20:19.536 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:19.536 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:19.536 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1914496378' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:19.536 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:19.537 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[105748]: pgmap v959: 99 pgs: 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 21728/176232 objects degraded (12.329%); 2625/176232 objects misplaced (1.490%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:20:19.537 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:19.537 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:19.537 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1914496378' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:19.537 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:19.542 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:19.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:19.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:19.684 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:19.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.724 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:19.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:19 smithi045 ceph-mon[95976]: pgmap v959: 99 pgs: 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 21728/176232 objects degraded (12.329%); 2625/176232 objects misplaced (1.490%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:20:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1914496378' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:19 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:19.824 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:19.862 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:20:19.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.867 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:19.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:19.946 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:20:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:20:19] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.51.0" 2024-09-21T10:20:19.969 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:19.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:20.020 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 44 16 63150 63134 89.6524 49.375 0.0104329 0.0111474 2024-09-21T10:20:20.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:20.122 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:20.149 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:20:20.224 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:20.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.270 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:20.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:20.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:20.409 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:20.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:20.425 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:20:20.425 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:20:20.425 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 0 2024-09-21T10:20:20.425 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:20:20.511 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:20.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.548 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:20.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:20.649 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:20.693 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:20.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:20.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:20 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:20:20.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:20 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:20.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:20 smithi032 ceph-mon[105748]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T10:20:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:20 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:20:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:20 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:20.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:20 smithi032 ceph-mon[101299]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T10:20:20.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:20 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:20:20.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:20 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:20.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:20 smithi045 ceph-mon[95976]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T10:20:20.795 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:20.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.837 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:20.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:20.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:20.972 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:20.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:21.020 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 45 16 63979 63963 88.8116 51.8125 0.0101828 0.0112528 2024-09-21T10:20:21.075 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:21.101 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:20:21.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:21.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:21.232 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:21.266 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:21.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:21.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:21.397 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:21.497 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:21.499 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:21.532 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:21.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:21.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:21.673 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:21.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:21.775 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:21.819 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:21.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:21.921 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:21.961 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:21.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:22.020 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 46 16 64916 64900 88.154 58.5625 0.00647784 0.0113342 2024-09-21T10:20:22.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:22.086 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:22.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:22.188 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:22.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:22.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:22.326 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:22.371 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:22.394 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:22 smithi045 ceph-mon[95976]: pgmap v961: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 33288/187254 objects degraded (17.777%); 1832/187254 objects misplaced (0.978%); 8.8 MiB/s, 141 objects/s recovering 2024-09-21T10:20:22.394 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 33288/187254 objects degraded (17.777%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:22 smithi032 ceph-mon[105748]: pgmap v961: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 33288/187254 objects degraded (17.777%); 1832/187254 objects misplaced (0.978%); 8.8 MiB/s, 141 objects/s recovering 2024-09-21T10:20:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 33288/187254 objects degraded (17.777%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:22 smithi032 ceph-mon[101299]: pgmap v961: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 33288/187254 objects degraded (17.777%); 1832/187254 objects misplaced (0.978%); 8.8 MiB/s, 141 objects/s recovering 2024-09-21T10:20:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 33288/187254 objects degraded (17.777%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:22.474 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:22.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:22.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:22.619 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:22.663 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:22.764 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:22.805 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:22.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.947 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:22.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:23.021 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 47 16 65674 65658 87.2864 47.375 0.0120718 0.0114506 2024-09-21T10:20:23.048 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:23.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:23.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:23.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.214 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:23.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:23.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:23.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:23.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:23.464 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:23.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:23.613 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:23.654 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:23.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:23.755 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:23.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.791 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:23.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:23.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:23.926 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:24.021 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 48 16 66653 66637 86.7427 61.1875 0.0126478 0.0115211 2024-09-21T10:20:24.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:24.078 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:24.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:24.180 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:24.183 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:20:24.184 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:20:24.184 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:20:24.184 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:20:24.184 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:20:24.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:24.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:24 smithi045 ceph-mon[95976]: pgmap v962: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 33288/188952 objects degraded (17.617%); 1572/188952 objects misplaced (0.832%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:20:24.327 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:24.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.363 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:24.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:24 smithi032 ceph-mon[105748]: pgmap v962: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 33288/188952 objects degraded (17.617%); 1572/188952 objects misplaced (0.832%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:20:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:24 smithi032 ceph-mon[101299]: pgmap v962: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 33288/188952 objects degraded (17.617%); 1572/188952 objects misplaced (0.832%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:20:24.464 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:24.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:24.603 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:24.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:24.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:24.669 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:24.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:24.802 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:24.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:24.903 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:24.930 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:20:24.948 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:24.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:25.021 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 49 16 67588 67572 86.1649 58.4375 0.0121622 0.0115999 2024-09-21T10:20:25.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:25.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.084 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:25.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:25.185 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:25.252 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:25.338 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:25.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:25.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.401 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:25.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:25.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:25.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:25.645 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:25.676 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:25.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:25.778 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:25.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.818 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:25.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:25.919 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:25.959 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:26.021 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 50 16 68514 68498 85.5991 57.875 0.0216667 0.0116755 2024-09-21T10:20:26.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:26.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:26.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:26.107 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:20:26.107 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:26.110 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:20:26.158 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:20:26.158 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:26.193 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:26.198 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:20:26.198 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:26.198 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:26 smithi032 ceph-mon[101299]: pgmap v963: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 33162/193791 objects degraded (17.112%); 1572/193791 objects misplaced (0.811%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:20:26.199 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2915042397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:26.199 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:26 smithi032 ceph-mon[105748]: pgmap v963: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 33162/193791 objects degraded (17.112%); 1572/193791 objects misplaced (0.811%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:20:26.199 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2915042397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:26.236 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:20:26.236 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:26.241 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:26.268 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:20:26.268 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:26.304 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:20:26.304 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:26.337 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:20:26.337 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:26.344 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:26.371 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:20:26.371 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:26.398 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:26.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:26.499 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:26 smithi045 ceph-mon[95976]: pgmap v963: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 33162/193791 objects degraded (17.112%); 1572/193791 objects misplaced (0.811%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:20:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2915042397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:26.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.541 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:26.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:26.641 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:26.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:26.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:26.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:26.817 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:26.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:26.919 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:26.957 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:27.021 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 51 16 69339 69323 84.9317 51.5625 0.0115906 0.0117684 2024-09-21T10:20:27.058 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:27.114 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:27.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:27.216 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:27.252 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:27.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:27.346 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:27.346 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:27.353 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:27.378 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:27.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:27.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:27 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 33162/193791 objects degraded (17.112%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:27.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:27 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 33162/193791 objects degraded (17.112%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:27.480 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:27.520 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:27.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:27 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 33162/193791 objects degraded (17.112%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:27.622 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:27.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:27.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:27.763 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:27.806 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:27.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:27.908 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:27.950 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:27.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:28.021 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 52 16 70121 70105 84.2382 48.875 0.00413116 0.0118553 2024-09-21T10:20:28.052 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:28.088 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:28.190 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:28.217 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:20:28.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:28.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:28.325 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.377 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:28.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:28 smithi032 ceph-mon[101299]: pgmap v964: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 33162/200379 objects degraded (16.550%); 1572/200379 objects misplaced (0.785%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:20:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:28 smithi032 ceph-mon[105748]: pgmap v964: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 33162/200379 objects degraded (16.550%); 1572/200379 objects misplaced (0.785%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:20:28.478 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:28.484 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:28 smithi045 ceph-mon[95976]: pgmap v964: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 33162/200379 objects degraded (16.550%); 1572/200379 objects misplaced (0.785%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:20:28.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:28.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:28.635 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:28.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:28.792 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:28.829 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:28.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:28.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:28.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:29.021 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 53 16 70993 70977 83.6771 54.5 0.00985367 0.0119447 2024-09-21T10:20:29.070 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:29.117 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:29.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:29.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:29.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.256 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:29.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:29.356 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:29.401 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:29.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:29.503 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:29.542 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:29.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:29.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:29.687 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:29.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:20:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:20:29] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.51.0" 2024-09-21T10:20:29.788 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:29.818 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:29.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:29.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:29.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.957 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:29.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:30.022 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 54 16 71932 71916 83.2143 58.6875 0.00836884 0.0120104 2024-09-21T10:20:30.058 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:30.087 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:30.189 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:30.223 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:30.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:30.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:30.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:30.389 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:30 smithi032 ceph-mon[105748]: pgmap v965: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 33162/201975 objects degraded (16.419%); 1220/201975 objects misplaced (0.604%); 6.3 MiB/s, 100 objects/s recovering 2024-09-21T10:20:30.390 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:30 smithi032 ceph-mon[101299]: pgmap v965: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 33162/201975 objects degraded (16.419%); 1220/201975 objects misplaced (0.604%); 6.3 MiB/s, 100 objects/s recovering 2024-09-21T10:20:30.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:30 smithi045 ceph-mon[95976]: pgmap v965: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 33162/201975 objects degraded (16.419%); 1220/201975 objects misplaced (0.604%); 6.3 MiB/s, 100 objects/s recovering 2024-09-21T10:20:30.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:30.507 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:30.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:30.604 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:30.604 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:30.608 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:30.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.654 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:30.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:30.664 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:30.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:30.802 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:30.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:30.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:30.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:31.021 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 55 16 72769 72753 82.6525 52.3125 0.00358707 0.0120907 2024-09-21T10:20:31.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:31.094 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:31.191 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:20:31.196 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:31.224 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:31 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2622858384' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:20:31.224 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:31 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:20:31.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:31 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2622858384' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:20:31.224 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:31 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:20:31.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.233 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:31.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:31.271 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:31 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2622858384' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:20:31.271 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:31 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:20:31.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:20:31.334 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:31.412 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:31.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:31.513 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:31.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.569 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:31.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:31.670 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:31.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:31.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:31.790 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:31.822 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:31.849 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:31.950 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:31.991 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:31.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:32.022 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 56 16 73555 73539 82.0538 49.125 0.02385 0.0121792 2024-09-21T10:20:32.092 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:20:32.093 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:32.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:32.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:32.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:32.235 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:32 smithi045 ceph-mon[95976]: pgmap v966: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 33036/213657 objects degraded (15.462%); 1220/213657 objects misplaced (0.571%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:20:32.235 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:32 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:20:32.235 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:32 smithi045 ceph-mon[95976]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T10:20:32.235 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:32 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 33036/213657 objects degraded (15.462%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:32.284 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:32.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:32.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:32.418 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:32.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:32 smithi032 ceph-mon[105748]: pgmap v966: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 33036/213657 objects degraded (15.462%); 1220/213657 objects misplaced (0.571%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:20:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:32 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:20:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:32 smithi032 ceph-mon[105748]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T10:20:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:32 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 33036/213657 objects degraded (15.462%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:32 smithi032 ceph-mon[101299]: pgmap v966: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 33036/213657 objects degraded (15.462%); 1220/213657 objects misplaced (0.571%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:20:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:32 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:20:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:32 smithi032 ceph-mon[101299]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T10:20:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:32 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 33036/213657 objects degraded (15.462%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:32.520 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:32.555 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:32.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:32.657 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:32.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:32.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:32.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.849 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:32.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:32.950 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:32.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.990 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:32.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:33.022 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 57 16 74421 74405 81.5638 54.125 0.0141711 0.0122537 2024-09-21T10:20:33.091 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.140 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:33.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:33.241 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.286 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:33.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:33.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:33.421 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:33.523 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:33 smithi045 ceph-mon[95976]: pgmap v968: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 33036/215118 objects degraded (15.357%); 856/215118 objects misplaced (0.398%); 6.0 MiB/s, 96 objects/s recovering 2024-09-21T10:20:33.548 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:33.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:33.650 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:33.685 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:33.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:33.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:33 smithi032 ceph-mon[101299]: pgmap v968: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 33036/215118 objects degraded (15.357%); 856/215118 objects misplaced (0.398%); 6.0 MiB/s, 96 objects/s recovering 2024-09-21T10:20:33.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:33 smithi032 ceph-mon[105748]: pgmap v968: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 33036/215118 objects degraded (15.357%); 856/215118 objects misplaced (0.398%); 6.0 MiB/s, 96 objects/s recovering 2024-09-21T10:20:33.786 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:33.818 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:20:33.830 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:33.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:33.859 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:33.859 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:33.932 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.962 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:33.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:34.026 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 58 16 75107 75091 80.8945 42.875 0.0211599 0.0123531 2024-09-21T10:20:34.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:34.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.111 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:34.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:34.211 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:34.242 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:34.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:34.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:34.381 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:34.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:34.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.517 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:34.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:34.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:20:34.618 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:34.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:34.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:34.766 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:34.793 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:34.793 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:34.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:34.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:34.904 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:34.947 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:34.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.955 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:34.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:34.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:34.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:35.024 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 59 16 75942 75926 80.4079 52.1875 0.0179098 0.01243 2024-09-21T10:20:35.055 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:35.089 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:35.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:35.190 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:35.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:35.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:35.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:35.355 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:35.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:35.457 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:35.480 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:35.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:35.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:35.604 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:35.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:35.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:35.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:35.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:35.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:35 smithi045 ceph-mon[95976]: pgmap v969: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 32899/219828 objects degraded (14.966%); 856/219828 objects misplaced (0.389%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:20:35.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:35.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:35 smithi032 ceph-mon[105748]: pgmap v969: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 32899/219828 objects degraded (14.966%); 856/219828 objects misplaced (0.389%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:20:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:35 smithi032 ceph-mon[101299]: pgmap v969: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 32899/219828 objects degraded (14.966%); 856/219828 objects misplaced (0.389%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:20:35.982 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:36.022 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:36.024 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:20:36.022734+0000 min lat: 0.00116053 max lat: 2.48922 avg lat: 0.0125426 2024-09-21T10:20:36.024 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:20:36.024 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 60 16 76538 76522 79.6887 37.25 0.00706443 0.0125426 2024-09-21T10:20:36.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:36.162 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:36.264 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:36.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.313 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:36.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:36.413 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:36.451 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:36.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:36.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:36.684 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:36.719 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:36.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:36.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:36.843 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:36.843 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:36.844 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:36.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:36.945 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:36.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:37.024 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 61 16 77323 77307 79.1867 49.0625 0.0310289 0.0126229 2024-09-21T10:20:37.081 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:37.116 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:37.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:37.218 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:37.251 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:37.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:37.353 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:37.397 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:37.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:37.499 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:37.531 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:37.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:37.633 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:37.678 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:37.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:37.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:37.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:37 smithi045 ceph-mon[95976]: pgmap v970: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s; 32899/225909 objects degraded (14.563%); 856/225909 objects misplaced (0.379%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:20:37.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:37 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 32899/225909 objects degraded (14.563%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:37.824 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:37.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:37.926 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:37.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:37 smithi032 ceph-mon[101299]: pgmap v970: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s; 32899/225909 objects degraded (14.563%); 856/225909 objects misplaced (0.379%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:20:37.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:37 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 32899/225909 objects degraded (14.563%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:37.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:37 smithi032 ceph-mon[105748]: pgmap v970: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s; 32899/225909 objects degraded (14.563%); 856/225909 objects misplaced (0.379%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:20:37.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:37 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 32899/225909 objects degraded (14.563%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:37.969 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:37.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:38.024 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 62 16 78132 78116 78.725 50.5625 0.00439105 0.0126927 2024-09-21T10:20:38.071 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:38.094 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:38.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:38.120 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:20:38.196 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:38.231 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:38.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:38.332 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:38.374 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:38.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:38.439 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:38.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:38.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:38.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:38.540 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:20:38.613 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.655 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:38.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:38.755 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:38.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:38 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1328372507' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:20:38.803 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:38.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:38.905 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:38.938 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:38.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:38.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:38 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1328372507' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:20:38.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:38 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1328372507' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:20:39.024 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 63 16 78848 78832 78.1857 44.75 0.0131027 0.0127826 2024-09-21T10:20:39.040 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:39.081 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:39.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:39.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:39.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:39.321 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:39.364 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:39.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:39.466 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:39.506 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:39.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:39.535 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:39.607 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:39.608 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:39.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.639 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:39.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:39.725 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:39 smithi032 ceph-mon[101299]: pgmap v971: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 32899/227364 objects degraded (14.470%); 426/227364 objects misplaced (0.187%); 6.6 MiB/s, 105 objects/s recovering 2024-09-21T10:20:39.725 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:39 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:39.725 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1328372507' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:20:39.725 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:39 smithi032 ceph-mon[101299]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T10:20:39.725 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:20:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:20:39] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.51.0" 2024-09-21T10:20:39.726 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:39 smithi032 ceph-mon[105748]: pgmap v971: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 32899/227364 objects degraded (14.470%); 426/227364 objects misplaced (0.187%); 6.6 MiB/s, 105 objects/s recovering 2024-09-21T10:20:39.726 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:39 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:39.726 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1328372507' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:20:39.726 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:39 smithi032 ceph-mon[105748]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T10:20:39.740 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:39.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:39.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:39 smithi045 ceph-mon[95976]: pgmap v971: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 32899/227364 objects degraded (14.470%); 426/227364 objects misplaced (0.187%); 6.6 MiB/s, 105 objects/s recovering 2024-09-21T10:20:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:39 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:39 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1328372507' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:20:39.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:39 smithi045 ceph-mon[95976]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T10:20:39.880 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:39.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:40.014 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:40.024 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 64 16 79747 79731 77.842 56.1875 0.00804667 0.0128406 2024-09-21T10:20:40.058 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:40.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:40.159 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:40.191 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:40.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:40.293 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:40.344 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:40.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:40.446 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:40.484 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:40.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:40.495 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:40.495 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:40.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:40.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:40.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:40.724 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:40.755 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:40.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:40.857 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:40.895 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:40.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:40.997 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:41.025 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 65 16 80624 80608 77.4867 54.8125 0.0307659 0.0128993 2024-09-21T10:20:41.034 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:41.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:41.135 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:41.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.172 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:41.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:41.240 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:20:41.274 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.335 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:41.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:41.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:41.477 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:41.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:41.498 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:20:41.578 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:41.605 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:41.609 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:41.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:41.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:41 smithi032 ceph-mon[101299]: pgmap v973: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 32713/237090 objects degraded (13.798%); 426/237090 objects misplaced (0.180%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:20:41.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:41 smithi032 ceph-mon[105748]: pgmap v973: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 32713/237090 objects degraded (13.798%); 426/237090 objects misplaced (0.180%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:20:41.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:41.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:41.790 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:41 smithi045 ceph-mon[95976]: pgmap v973: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 32713/237090 objects degraded (13.798%); 426/237090 objects misplaced (0.180%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:20:41.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:41.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.869 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:41.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:41.970 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:41.996 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:41.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:42.025 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 66 16 81233 81217 76.8894 38.0625 0.00493171 0.012932 2024-09-21T10:20:42.098 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:42.122 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:42.223 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:42.246 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:42.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:42.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:42.371 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:42.473 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:42.496 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:42.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:42.598 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:42.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:42.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:42.724 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:42.762 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:42.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:42 smithi045 ceph-mon[95976]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T10:20:42.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:42 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 32713/237090 objects degraded (13.798%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:42.864 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:42.905 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:42.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:42 smithi032 ceph-mon[101299]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T10:20:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:42 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 32713/237090 objects degraded (13.798%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:42 smithi032 ceph-mon[105748]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T10:20:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:42 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 32713/237090 objects degraded (13.798%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:43.007 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:43.026 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 67 16 81536 81520 76.0246 18.9375 0.0052412 0.0131393 2024-09-21T10:20:43.029 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:43.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:43.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:43.176 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:43.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:43.278 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:43.313 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:43.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:43.415 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:43.439 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:43.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:43.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:43.578 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:43.680 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:43.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:43 smithi045 ceph-mon[95976]: pgmap v975: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 32713/238527 objects degraded (13.715%); 8.1 MiB/s, 130 objects/s recovering 2024-09-21T10:20:43.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:43 smithi045 ceph-mon[95976]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T10:20:43.818 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:43.840 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:43 smithi032 ceph-mon[105748]: pgmap v975: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 32713/238527 objects degraded (13.715%); 8.1 MiB/s, 130 objects/s recovering 2024-09-21T10:20:43.840 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:43 smithi032 ceph-mon[105748]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T10:20:43.841 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:43 smithi032 ceph-mon[101299]: pgmap v975: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 32713/238527 objects degraded (13.715%); 8.1 MiB/s, 130 objects/s recovering 2024-09-21T10:20:43.841 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:43 smithi032 ceph-mon[101299]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T10:20:43.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.857 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:43.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:43.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:43.981 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:44.025 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 68 16 82237 82221 75.5509 43.8125 0.00973701 0.0132253 2024-09-21T10:20:44.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:44.112 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:44.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:44.214 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:44.245 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:44.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:44.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:44.377 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:44.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:44.479 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:44.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:44.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:44.624 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:20:44.624 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:44.625 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:44.626 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:44.654 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:44.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:44.755 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:44.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:44.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:44.900 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:44.923 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:44.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:44.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:44 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3690831115' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:44.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:44 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3690831115' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:45.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:45.025 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 69 16 83034 83018 75.1779 49.8125 0.00354125 0.0132953 2024-09-21T10:20:45.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:44 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3690831115' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:45.058 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:45.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:45.160 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:45.203 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:45.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:45.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:45.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:45.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:45.460 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:45.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:45.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:45.651 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:45.710 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:45 smithi045 ceph-mon[95976]: pgmap v977: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 31838/242574 objects degraded (13.125%); 287/242574 objects misplaced (0.118%); 12 MiB/s, 199 objects/s recovering 2024-09-21T10:20:45.753 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:45.786 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:45.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:45.929 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:45.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:45.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:45 smithi032 ceph-mon[101299]: pgmap v977: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 31838/242574 objects degraded (13.125%); 287/242574 objects misplaced (0.118%); 12 MiB/s, 199 objects/s recovering 2024-09-21T10:20:45.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:45 smithi032 ceph-mon[105748]: pgmap v977: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 31838/242574 objects degraded (13.125%); 287/242574 objects misplaced (0.118%); 12 MiB/s, 199 objects/s recovering 2024-09-21T10:20:46.025 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 70 16 83890 83874 74.8682 53.5 0.0242136 0.0133505 2024-09-21T10:20:46.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:46.060 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:46.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:46.195 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:20:46.195 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:20:46.195 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:20:46.196 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:20:46.196 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:20:46.207 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:46.309 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:46.344 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:46.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:46.446 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:46.490 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:46.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:46.550 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:46.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:46.624 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:46.727 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:46.763 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:46.853 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:20:46.865 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:46.891 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:46.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:47.018 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:47.025 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 71 16 84535 84519 74.3815 40.3125 0.00346277 0.0133866 2024-09-21T10:20:47.084 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:47.120 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:47.142 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:47.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:47.244 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:47.268 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:47.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:47.369 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:47.392 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:47.494 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:47.516 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:47.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:47.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:47.663 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:47.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:47 smithi032 ceph-mon[105748]: pgmap v978: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 31838/247557 objects degraded (12.861%); 287/247557 objects misplaced (0.116%); 9.4 MiB/s, 151 objects/s recovering 2024-09-21T10:20:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:47 smithi032 ceph-mon[105748]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T10:20:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:47 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 31838/247557 objects degraded (12.861%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:47.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:47 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4258273202' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:20:47.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:47 smithi032 ceph-mon[101299]: pgmap v978: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 31838/247557 objects degraded (12.861%); 287/247557 objects misplaced (0.116%); 9.4 MiB/s, 151 objects/s recovering 2024-09-21T10:20:47.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:47 smithi032 ceph-mon[101299]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T10:20:47.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:47 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 31838/247557 objects degraded (12.861%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:47.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:47 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4258273202' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:20:47.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:47 smithi045 ceph-mon[95976]: pgmap v978: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 31838/247557 objects degraded (12.861%); 287/247557 objects misplaced (0.116%); 9.4 MiB/s, 151 objects/s recovering 2024-09-21T10:20:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:47 smithi045 ceph-mon[95976]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T10:20:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:47 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 31838/247557 objects degraded (12.861%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:47.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:47 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4258273202' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:20:47.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.798 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:47.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:47.899 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:47.936 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:48.025 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 72 16 84927 84911 73.6888 24.5 0.0244904 0.0135647 2024-09-21T10:20:48.038 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:48.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:48.177 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.211 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:48.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:48.251 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:20:48.312 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:48.353 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:48.455 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:48.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:48.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:48.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:20:48.600 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:48.645 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:48.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:48.747 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:48.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:48.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:48.878 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:48.879 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:48 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4258273202' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:20:48.879 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:48 smithi045 ceph-mon[95976]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T10:20:48.879 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:48.879 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:48.930 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:48.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:48.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:48 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4258273202' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:20:48.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:48 smithi032 ceph-mon[105748]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T10:20:48.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:48.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:48.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:48 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4258273202' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:20:48.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:48 smithi032 ceph-mon[101299]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T10:20:48.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:20:48.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:49.025 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 73 16 85787 85771 73.4157 53.75 0.0149755 0.0136151 2024-09-21T10:20:49.033 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:49.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:20:49.065 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:20:49.065 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:20:49.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:20:49.065 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:20:49.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.065 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:58192 8.8 8.380c70b8 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e324)", 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:12.090886+0000", 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 36.963534371000001, 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.980038615, 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 58192 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.066 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.090886+0000", 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.090886+0000", 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.090885+0000", 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091022+0000", 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013705999999999999 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091041+0000", 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9128e-05 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091048+0000", 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4389999999999997e-06 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091094+0000", 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6360999999999999e-05 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091112+0000", 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7363999999999998e-05 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.070913+0000", 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97980165600000002 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.070925+0000", 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1874e-05 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:58199 8.8 8.549d6668 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e324)", 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:12.091060+0000", 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 36.963360840999997, 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98072915800000005, 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:20:49.070 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 58199 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091060+0000", 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091060+0000", 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091060+0000", 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091129+0000", 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9474999999999995e-05 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091139+0000", 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0451e-05 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091146+0000", 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0110000000000003e-06 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091187+0000", 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0142999999999998e-05 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:20:49.073 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091197+0000", 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.9429999999999995e-06 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.071780+0000", 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98058355100000005 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.071789+0000", 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0629999999999995e-06 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.074 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:58202 8.d 8.cb2b9c6d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:12.091156+0000", 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 36.963265270000001, 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98421055999999996, 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 58202 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.075 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091156+0000", 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091156+0000", 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091155+0000", 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091207+0000", 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1714e-05 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091217+0000", 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0404999999999999e-05 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091223+0000", 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9649999999999996e-06 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091263+0000", 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9734999999999998e-05 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.091270+0000", 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7299999999999999e-06 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.075357+0000", 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98408782299999997 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.075366+0000", 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6920000000000008e-06 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:58213 8.8 8.ca2941f8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:12.111561+0000", 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 36.942859789000003, 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96187348399999995, 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:20:49.079 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 58213 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.111561+0000", 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.111561+0000", 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.111559+0000", 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.111622+0000", 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2791000000000002e-05 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.111638+0000", 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5310000000000001e-05 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.111645+0000", 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2080000000000003e-06 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.111688+0000", 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3028999999999997e-05 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.111693+0000", 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2800000000000003e-06 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.073382+0000", 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96168879200000001 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.073435+0000", 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2686999999999999e-05 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:58215 8.d 8.eee6accd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:12.113403+0000", 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 36.941017995000003, 2024-09-21T10:20:49.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96203652799999995, 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 58215 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.113403+0000", 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.113403+0000", 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.113402+0000", 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.113466+0000", 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4035000000000006e-05 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.113477+0000", 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1248e-05 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.113483+0000", 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5629999999999999e-06 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.113522+0000", 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9056e-05 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:12.113528+0000", 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4810000000000003e-06 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.075427+0000", 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96189850700000001 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:13.075439+0000", 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2514000000000001e-05 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.087 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:58201 8.a e326/325 8:5b7498c1:::benchmark_data_smithi032_233595_object58200:head v 326'3546, mlcod=0'0)", 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:14.478653+0000", 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 34.575767818999999, 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.062881240000000005, 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.478653+0000", 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.478653+0000", 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.478652+0000", 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.478690+0000", 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7761e-05 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.478701+0000", 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0913999999999999e-05 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.478709+0000", 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3000000000000002e-06 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.478819+0000", 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010956299999999999 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.478842+0000", 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3402000000000001e-05 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.541503+0000", 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.062660918999999995 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:14.541534+0000", 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1192999999999998e-05 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.091 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "MOSDPGPush(8.8 326/325 [PushOp(8:120f696b:::benchmark_data_smithi032_233595_object1565:head, version: 313'97, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:120f696b:::benchmark_data_smithi032_233595_object1565:head@313'97, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:17.768417+0000", 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 31.286003969999999, 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.058928636, 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.768417+0000", 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.768417+0000", 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.768416+0000", 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.768459+0000", 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.354e-05 2024-09-21T10:20:49.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.768481+0000", 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1420000000000002e-05 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.768499+0000", 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7968000000000001e-05 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.825906+0000", 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.057407317999999999 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.825923+0000", 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6429000000000001e-05 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.827346+0000", 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001422909 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.095 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:61324 8.0 e326/325 8:0b9ce778:::benchmark_data_smithi032_233595_object61323:head v 326'3867, mlcod=0'0)", 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:17.783362+0000", 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 31.271058492000002, 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.061971189000000003, 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.783362+0000", 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.783362+0000", 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.783362+0000", 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.783428+0000", 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6663999999999997e-05 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.783445+0000", 2024-09-21T10:20:49.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6460000000000002e-05 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.783452+0000", 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3490000000000003e-06 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.789829+0000", 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0063763980000000001 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.789853+0000", 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4595000000000001e-05 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.845257+0000", 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.055404255999999999 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.845334+0000", 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6100000000000007e-05 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:61331 8.0 e326/325 8:0d0ed4f7:::benchmark_data_smithi032_233595_object61330:head v 326'3868, mlcod=0'0)", 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:17.789886+0000", 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 31.264534964999999, 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.055511448999999997, 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.789886+0000", 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.789886+0000", 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.789885+0000", 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.789938+0000", 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3362999999999997e-05 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.789948+0000", 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4949999999999997e-06 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.789954+0000", 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1890000000000004e-06 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.804614+0000", 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014660171 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.804733+0000", 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011877799999999999 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.845338+0000", 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.040605024000000003 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.845397+0000", 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9222999999999999e-05 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.103 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:61339 8.8 e326/325 8:1e86108f:::benchmark_data_smithi032_233595_object61338:head v 326'3852, mlcod=313'96)", 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:17.798827+0000", 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 31.255593417, 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.053332323000000001, 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.798827+0000", 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.798827+0000", 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.798827+0000", 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.798923+0000", 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5947999999999997e-05 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.798934+0000", 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1547e-05 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.798940+0000", 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2740000000000004e-06 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.802142+0000", 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00320112 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.802162+0000", 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0483e-05 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.852125+0000", 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.049963331999999999 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:17.852160+0000", 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4334e-05 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:64894 8.4 e327/325 8:210207c9:::benchmark_data_smithi032_233595_object64893:head v 327'4049, mlcod=0'0)", 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:21.988617+0000", 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 27.065803889000001, 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16495448200000001, 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.988617+0000", 2024-09-21T10:20:49.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.988617+0000", 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.988616+0000", 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.988645+0000", 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8815000000000001e-05 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.988657+0000", 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2048000000000001e-05 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.988664+0000", 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0690000000000004e-06 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.988710+0000", 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5630999999999997e-05 2024-09-21T10:20:49.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.988732+0000", 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2476999999999998e-05 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.153530+0000", 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16479765499999999 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.153571+0000", 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1798999999999997e-05 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.111 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:64892 8.3 e327/325 8:cb1ac685:::benchmark_data_smithi032_233595_object64891:head v 327'4034, mlcod=0'0)", 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:21.990793+0000", 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 27.063627758999999, 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15556689800000001, 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.990793+0000", 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.990793+0000", 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.990792+0000", 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.990901+0000", 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000108571 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.990915+0000", 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3903000000000001e-05 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.990922+0000", 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7870000000000002e-06 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.990990+0000", 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7205000000000005e-05 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.991014+0000", 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4182e-05 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.146314+0000", 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.155299944 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.146360+0000", 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6236999999999998e-05 2024-09-21T10:20:49.115 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:64900 8.4 e327/325 8:24c96567:::benchmark_data_smithi032_233595_object64899:head v 327'4050, mlcod=0'0)", 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:21.991381+0000", 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 27.063040137000002, 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.162240946, 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.116 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.117 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:49.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:49.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.991381+0000", 2024-09-21T10:20:49.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.991381+0000", 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.991380+0000", 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.991731+0000", 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00035125899999999998 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.991742+0000", 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1172000000000001e-05 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.991749+0000", 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2910000000000004e-06 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992970+0000", 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0012209720000000001 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992993+0000", 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3277000000000001e-05 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.153578+0000", 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.160584793 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.153622+0000", 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3917e-05 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.121 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:64896 8.a e327/325 8:52b4b1d6:::benchmark_data_smithi032_233595_object64895:head v 327'3900, mlcod=0'0)", 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:21.992325+0000", 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 27.062095477, 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.156376397, 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992325+0000", 2024-09-21T10:20:49.122 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992325+0000", 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992324+0000", 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992367+0000", 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2330000000000003e-05 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992384+0000", 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7541e-05 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992391+0000", 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7170000000000002e-06 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992851+0000", 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00046039799999999999 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.992902+0000", 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0158999999999999e-05 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.148589+0000", 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15568710099999999 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.148702+0000", 2024-09-21T10:20:49.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000113162 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:64897 8.8 e327/325 8:13e98114:::benchmark_data_smithi032_233595_object64896:head v 327'4099, mlcod=313'198)", 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:21.993032+0000", 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 27.061388988000001, 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16826672000000001, 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993032+0000", 2024-09-21T10:20:49.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993032+0000", 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993031+0000", 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993876+0000", 2024-09-21T10:20:49.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00084489000000000005 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993892+0000", 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5590000000000002e-05 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993900+0000", 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2539999999999992e-06 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.994049+0000", 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014951799999999999 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.994080+0000", 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0574000000000002e-05 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.161252+0000", 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16717162799999999 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.161299+0000", 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7055999999999998e-05 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.129 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "MOSDPGPush(8.8 327/325 [PushOp(8:1976e172:::benchmark_data_smithi032_233595_object3394:head, version: 313'201, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:1976e172:::benchmark_data_smithi032_233595_object3394:head@313'201, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:21.993916+0000", 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 27.060504472000002, 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.156741731, 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993916+0000", 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993916+0000", 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993916+0000", 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993963+0000", 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7132000000000002e-05 2024-09-21T10:20:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993977+0000", 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.381e-05 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.993988+0000", 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1586000000000001e-05 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.149039+0000", 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.155051051 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.149052+0000", 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2079000000000001e-05 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.150658+0000", 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001606594 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:64901 8.d e327/325 8:bf84974b:::benchmark_data_smithi032_233595_object64900:head v 327'4069, mlcod=0'0)", 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:21.995389+0000", 2024-09-21T10:20:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 27.059031446999999, 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16891655799999999, 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.995389+0000", 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.995389+0000", 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.995388+0000", 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.995431+0000", 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3222000000000001e-05 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145910+0000", 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15047920400000001 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145922+0000", 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1333e-05 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145974+0000", 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2064999999999997e-05 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145993+0000", 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9500999999999998e-05 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.164139+0000", 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018145762999999999 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.164306+0000", 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016683400000000001 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:64903 8.a e327/325 8:51816d80:::benchmark_data_smithi032_233595_object64902:head v 327'3901, mlcod=0'0)", 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:21.995526+0000", 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 27.058894549000001, 2024-09-21T10:20:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15813791799999999, 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.995526+0000", 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.995526+0000", 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.995526+0000", 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.995596+0000", 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0419999999999993e-05 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145915+0000", 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15031892599999999 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145924+0000", 2024-09-21T10:20:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.1970000000000002e-06 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.148729+0000", 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002805054 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.148748+0000", 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.863e-05 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.153564+0000", 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0048161410000000003 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.153664+0000", 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000100221 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:64907 8.9 e327/325 8:95b88cd9:::benchmark_data_smithi032_233595_object64906:head v 327'4088, mlcod=0'0)", 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:21.997859+0000", 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 27.056561585000001, 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15853286, 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.997859+0000", 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.997859+0000", 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.997858+0000", 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:21.997880+0000", 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1639999999999999e-05 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145897+0000", 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14801686999999999 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145911+0000", 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.417e-05 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145958+0000", 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6872000000000001e-05 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.145990+0000", 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1578000000000002e-05 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.156258+0000", 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010268380000000001 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:22.156392+0000", 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013422 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:83406 8.c e331/325 8:35e26b6d:::benchmark_data_smithi032_233595_object83405:head v 331'5271, mlcod=0'0)", 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:20:45.474329+0000", 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 3.580092021, 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.067375004000000002, 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.474329+0000", 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.474329+0000", 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.474328+0000", 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:20:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.474380+0000", 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2704000000000002e-05 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.474392+0000", 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1997000000000001e-05 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.474400+0000", 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2509999999999997e-06 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.492461+0000", 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018061648999999999 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.492489+0000", 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7424999999999999e-05 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.541655+0000", 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.049166543 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:20:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:20:45.541704+0000", 2024-09-21T10:20:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8501000000000002e-05 2024-09-21T10:20:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:20:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:20:49.149 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:20:49.219 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:49.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.271 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:49.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:49.372 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:49.410 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:49.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:49.511 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:49.552 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:49.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:49.594 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:20:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:20:49] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-09-21T10:20:49.655 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:49.730 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:49.733 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:49.835 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:49.867 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:49.890 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:49 smithi032 ceph-mon[105748]: pgmap v981: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 571 op/s; 31838/248814 objects degraded (12.796%); 287/248814 objects misplaced (0.115%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:20:49.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:49 smithi032 ceph-mon[101299]: pgmap v981: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 571 op/s; 31838/248814 objects degraded (12.796%); 287/248814 objects misplaced (0.115%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:20:49.969 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:50.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.017 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:50.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:50.028 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 74 16 86565 86549 73.0783 48.625 0.0274708 0.0136774 2024-09-21T10:20:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:49 smithi045 ceph-mon[95976]: pgmap v981: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 571 op/s; 31838/248814 objects degraded (12.796%); 287/248814 objects misplaced (0.115%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:20:50.118 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:50.165 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:50.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:20:50.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:50.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:50.405 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:50.435 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:50.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:50.537 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:50.576 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:50.616 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:20:50.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:50.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:50.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:50.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:50.864 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:50.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:50 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2049861353' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:50 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:50 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2049861353' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:50 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:50.966 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:51.008 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:51.028 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 75 16 87354 87338 72.7615 49.3125 0.0111937 0.0137367 2024-09-21T10:20:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:50 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2049861353' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:50 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:51.110 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:51.134 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:51.235 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:51.278 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:51.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:51.380 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:51.413 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:51.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:51.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:51.557 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:51.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:51.659 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:51.701 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:51.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:51.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:51.831 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:51.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:51.933 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:51 smithi032 ceph-mon[105748]: pgmap v982: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 31556/257898 objects degraded (12.236%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T10:20:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:51 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:51 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:51 smithi032 ceph-mon[105748]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T10:20:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:51 smithi032 ceph-mon[101299]: pgmap v982: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 31556/257898 objects degraded (12.236%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T10:20:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:51 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:51 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:51.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:51 smithi032 ceph-mon[101299]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T10:20:51.977 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:52.028 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 76 16 88213 88197 72.5105 53.6875 0.0174902 0.0137853 2024-09-21T10:20:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:51 smithi045 ceph-mon[95976]: pgmap v982: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 31556/257898 objects degraded (12.236%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T10:20:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:51 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:51 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:51 smithi045 ceph-mon[95976]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T10:20:52.080 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:52.121 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:52.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:52.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:52.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:52.361 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:52.399 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:52.470 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:52.501 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.553 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:52.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:52.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:52.690 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:52.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:52.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:52.834 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:52.936 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:52.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:52 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 31556/257898 objects degraded (12.236%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:52.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:52 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 31556/257898 objects degraded (12.236%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:52.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:53.028 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 77 16 89118 89102 72.3033 56.5625 0.0180161 0.0138235 2024-09-21T10:20:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:52 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 31556/257898 objects degraded (12.236%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:53.081 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:53.115 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:53.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:53.217 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.252 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:53.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:53.353 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:53.385 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:53.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:53.487 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:53.523 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:53.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:53.624 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:53.657 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:53.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:53.759 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:53.795 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:53.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:53.828 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:53 smithi032 ceph-mon[101299]: pgmap v984: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s; 31556/259215 objects degraded (12.174%); 5.9 MiB/s, 94 objects/s recovering 2024-09-21T10:20:53.828 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:53 smithi032 ceph-mon[105748]: pgmap v984: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s; 31556/259215 objects degraded (12.174%); 5.9 MiB/s, 94 objects/s recovering 2024-09-21T10:20:53.897 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:53.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:53.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:54.028 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 78 16 90051 90035 72.1239 58.3125 0.0268377 0.0138586 2024-09-21T10:20:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:53 smithi045 ceph-mon[95976]: pgmap v984: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s; 31556/259215 objects degraded (12.174%); 5.9 MiB/s, 94 objects/s recovering 2024-09-21T10:20:54.052 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:54.090 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:54.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:54.192 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:54.242 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:54.344 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:54.378 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:54.480 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:54.519 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:54.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:54.621 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:54.643 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:54.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:54.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:54.789 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:54.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:54.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:54.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:54.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:55.028 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 79 16 90871 90855 71.8597 51.25 0.0274568 0.0139092 2024-09-21T10:20:55.030 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:55.072 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:55.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:55.174 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:55.213 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:55.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:55.281 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:20:55.315 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:55.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:55.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:55.449 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:55.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:55.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:55.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:55.636 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:20:55.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.650 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:55.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:55.751 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:55.792 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:55.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:55.894 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:55.939 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:55.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:55 smithi032 ceph-mon[105748]: pgmap v985: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 31409/263991 objects degraded (11.898%); 5.7 MiB/s, 90 objects/s recovering 2024-09-21T10:20:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/739002386' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:20:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:55 smithi032 ceph-mon[101299]: pgmap v985: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 31409/263991 objects degraded (11.898%); 5.7 MiB/s, 90 objects/s recovering 2024-09-21T10:20:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/739002386' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:20:56.028 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:20:56.027667+0000 min lat: 0.00116053 max lat: 2.48922 avg lat: 0.0139513 2024-09-21T10:20:56.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:20:56.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 80 16 91748 91732 71.6466 54.8125 0.0137669 0.0139513 2024-09-21T10:20:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:55 smithi045 ceph-mon[95976]: pgmap v985: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 31409/263991 objects degraded (11.898%); 5.7 MiB/s, 90 objects/s recovering 2024-09-21T10:20:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/739002386' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:20:56.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:56.074 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:56.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:56.176 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:56.217 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:56.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:56.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:56.353 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:56.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:56.432 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:56.454 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:56.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.489 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:56.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:56.590 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:56.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:56.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:56.726 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:56.768 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:56.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:56.870 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:56.910 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:56.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/739002386' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:20:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:56 smithi032 ceph-mon[105748]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T10:20:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/739002386' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:20:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:56 smithi032 ceph-mon[101299]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T10:20:57.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:57.021 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:57.025 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:20:57.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 81 16 92642 92626 71.4518 55.875 0.0347898 0.0139899 2024-09-21T10:20:57.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/739002386' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:20:57.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:56 smithi045 ceph-mon[95976]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T10:20:57.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:57.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:57.176 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:57.239 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:57.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:57.342 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:57.385 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:57.487 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:57.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.541 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:57.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:57.642 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:57.683 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:57.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:57.785 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:57.820 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:57.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:57.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:57 smithi032 ceph-mon[105748]: pgmap v987: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 31409/270573 objects degraded (11.608%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:20:57.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:57 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 31409/270573 objects degraded (11.608%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:57.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:57 smithi032 ceph-mon[101299]: pgmap v987: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 31409/270573 objects degraded (11.608%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:20:57.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:57 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 31409/270573 objects degraded (11.608%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:57.922 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:57.970 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:57.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:58.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 82 16 93491 93475 71.2275 53.0625 0.0514253 0.0140336 2024-09-21T10:20:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:57 smithi045 ceph-mon[95976]: pgmap v987: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 31409/270573 objects degraded (11.608%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:20:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:57 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 31409/270573 objects degraded (11.608%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:20:58.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:58.112 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:58.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:58.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:58.253 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:58.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:58.354 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:58.391 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:58.493 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:58.535 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:58.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:58.637 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.677 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:58.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:58.778 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:58.815 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:58.917 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:58.953 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:58.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:59.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 83 16 94389 94373 71.0456 56.125 0.00605049 0.0140692 2024-09-21T10:20:59.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:59.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.093 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:59.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:59.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:59.232 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:59.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:59.334 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:59.380 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:59.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:59.527 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:59.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:59.628 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:59.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:20:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:59.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:59.808 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:59.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:59.909 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:59.947 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:20:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:59.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:20:59 smithi032 ceph-mon[101299]: pgmap v988: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 601 op/s; 31409/272328 objects degraded (11.534%); 1.1 MiB/s, 18 objects/s recovering 2024-09-21T10:20:59.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:20:59 smithi032 ceph-mon[105748]: pgmap v988: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 601 op/s; 31409/272328 objects degraded (11.534%); 1.1 MiB/s, 18 objects/s recovering 2024-09-21T10:20:59.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:20:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:20:59] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-09-21T10:21:00.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 84 16 95304 95288 70.8805 57.1875 0.0163368 0.0141027 2024-09-21T10:21:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:20:59 smithi045 ceph-mon[95976]: pgmap v988: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 601 op/s; 31409/272328 objects degraded (11.534%); 1.1 MiB/s, 18 objects/s recovering 2024-09-21T10:21:00.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:00.090 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:00.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:00.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:00.233 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:00.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:00.335 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:00.392 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:00.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:00.494 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:00.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:00.617 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:00.651 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:00.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:00.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:00.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.791 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:00.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:00.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:00.933 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:01.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 85 16 96120 96104 70.6466 51 0.0108812 0.0141483 2024-09-21T10:21:01.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:01.067 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:01.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:01.169 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.211 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:01.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:01.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:01.353 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:01.454 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:01.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.488 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:01.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:01.589 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:01.636 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:01.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:01.737 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:01.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:01.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:01.829 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:01 smithi032 ceph-mon[105748]: pgmap v989: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s; 31224/283662 objects degraded (11.007%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:21:01.830 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:01 smithi032 ceph-mon[101299]: pgmap v989: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s; 31224/283662 objects degraded (11.007%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:21:01.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:01.900 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:01.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:02.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:02.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 86 16 96864 96848 70.3659 46.5 0.121773 0.0142026 2024-09-21T10:21:02.030 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:02.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:01 smithi045 ceph-mon[95976]: pgmap v989: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s; 31224/283662 objects degraded (11.007%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:21:02.132 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:02.165 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:02.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:02.266 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:02.305 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:02.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:02.407 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:02.431 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:02.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:02.533 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:02.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:02.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:02.666 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:02.691 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:02.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:02.793 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:02.832 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:02.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:02.930 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:21:02.930 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:21:02.930 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:21:02.931 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:21:02.931 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:02.931 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:02.931 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:02.931 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:02.931 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:02.931 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:21:02.931 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:21:02.931 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:21:02.932 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:21:02.932 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:02.932 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:02.932 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:02.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:02 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 31224/283662 objects degraded (11.007%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:21:02.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:02 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3551985924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:02.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:02 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 31224/283662 objects degraded (11.007%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:21:02.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:02 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3551985924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:02.934 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.971 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:02.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:03.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 87 16 97487 97471 70.0046 38.9375 0.0042375 0.014273 2024-09-21T10:21:03.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:02 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 31224/283662 objects degraded (11.007%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:21:03.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:02 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3551985924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:03.072 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:03.111 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:03.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:03.256 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:03.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:03.358 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:03.396 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:03.498 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:03.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:03.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:03.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:03.676 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:03.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:03.778 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:03.814 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:03.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:03.916 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:03.954 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:03 smithi032 ceph-mon[101299]: pgmap v990: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 31224/285366 objects degraded (10.942%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:21:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:03 smithi032 ceph-mon[105748]: pgmap v990: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 31224/285366 objects degraded (10.942%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:21:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:04.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 88 16 98330 98314 69.8078 52.6875 0.0230034 0.0143186 2024-09-21T10:21:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:03 smithi045 ceph-mon[95976]: pgmap v990: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 31224/285366 objects degraded (10.942%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:21:04.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:04.056 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:04.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:04.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:04.197 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:04.232 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:04.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:04.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:04.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:04.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:04.459 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:04.482 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:04.583 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:04.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:04.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:04.724 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:04.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.764 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:04.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:04.864 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:04.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:04.997 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:05.032 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 89 16 98967 98951 69.4708 39.8125 0.0362055 0.0143885 2024-09-21T10:21:05.044 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:05.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:05.147 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:05.186 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:05.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:05.288 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:05.322 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:05.424 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:05.457 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:05.559 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:05.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:05.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:05.694 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:05.748 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:05.783 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:05 smithi032 ceph-mon[105748]: pgmap v991: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 31026/289773 objects degraded (10.707%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:21:05.783 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:05 smithi032 ceph-mon[101299]: pgmap v991: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 31026/289773 objects degraded (10.707%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:21:05.850 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:05.883 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:05.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:05.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:06.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:06.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:06.029 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total time run: 90.0206 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total writes made: 99842 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Write size: 65536 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object size: 65536 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Bandwidth (MB/sec): 69.3188 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Bandwidth: 39.7148 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max bandwidth (MB/sec): 211.312 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average IOPS: 1109 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev IOPS: 635.445 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max IOPS: 3381 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min IOPS: 0 2024-09-21T10:21:06.030 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average Latency(s): 0.0144201 2024-09-21T10:21:06.031 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Latency(s): 0.0474049 2024-09-21T10:21:06.031 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max latency(s): 2.48922 2024-09-21T10:21:06.031 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min latency(s): 0.00116053 2024-09-21T10:21:06.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:05 smithi045 ceph-mon[95976]: pgmap v991: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 31026/289773 objects degraded (10.707%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:21:06.036 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:21:06.124 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:06.164 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:06.224 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:06.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:06.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.303 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:06.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:06.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:06.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:06.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:06.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:06.578 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:06.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:06.680 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:06.702 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:06.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:06.727 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:21:06.804 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:06.827 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:06.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:06.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:06.953 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:06.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3619221621' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:06.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:06 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:06.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3619221621' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:06.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:06 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3619221621' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:06 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:07.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:07.088 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:07.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:07.190 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:07.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:07.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:07.324 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:07.355 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:07.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:07.457 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:07.491 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:07.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:07.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:21:07.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:07.636 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:07.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:07.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:07.775 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:07.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:07.877 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:07.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:07.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[101299]: pgmap v992: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 31026/295188 objects degraded (10.511%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[101299]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 31026/295188 objects degraded (10.511%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[105748]: pgmap v992: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 31026/295188 objects degraded (10.511%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[105748]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T10:21:07.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:07 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 31026/295188 objects degraded (10.511%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:21:08.019 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:07 smithi045 ceph-mon[95976]: pgmap v992: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 31026/295188 objects degraded (10.511%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:21:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:07 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:21:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:07 smithi045 ceph-mon[95976]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T10:21:08.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:07 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 31026/295188 objects degraded (10.511%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:21:08.059 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:08.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:08.161 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:08.191 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:08.292 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:08.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:08.437 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:08.469 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:08.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:08.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:08.605 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:08.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:08.707 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:08.740 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:08.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:08.841 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:08.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:08.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:08.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.032 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:09.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:09.133 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:09.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:09.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:09.263 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:09.313 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:09.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:09.414 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:09.458 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:09.559 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:09.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.594 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:09.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:09.607 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:21:09.607 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:21:09.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:09.731 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:09.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:09.833 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:09.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:09.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:09.954 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:09 smithi032 ceph-mon[105748]: pgmap v994: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 811 op/s; 31026/296265 objects degraded (10.472%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:21:09.954 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:09 smithi032 ceph-mon[101299]: pgmap v994: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 811 op/s; 31026/296265 objects degraded (10.472%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:21:09.954 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:21:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:21:09] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.51.0" 2024-09-21T10:21:09.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:10.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:10.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:10.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:09 smithi045 ceph-mon[95976]: pgmap v994: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 811 op/s; 31026/296265 objects degraded (10.472%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:21:10.132 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:10.169 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:10.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:10.271 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:10.300 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:10.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:10.402 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:10.427 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:21:10.434 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:10.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:10.437 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:10.537 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:10.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:10.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:10.691 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:10.730 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:10.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:10.832 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:10.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:21:10.872 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:10.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:10.974 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:11.014 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:11.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:11.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:11.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:11.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:11.255 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:11.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:11.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:11.406 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:11.451 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:11.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:11.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:11.700 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:11.732 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:11.833 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:11.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:11.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:11 smithi032 ceph-mon[105748]: pgmap v995: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 775 op/s; 30749/292290 objects degraded (10.520%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:11.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:11 smithi032 ceph-mon[101299]: pgmap v995: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 775 op/s; 30749/292290 objects degraded (10.520%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:11.975 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.012 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:12.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:11 smithi045 ceph-mon[95976]: pgmap v995: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 775 op/s; 30749/292290 objects degraded (10.520%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:12.113 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:12.152 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:12.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:12.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:12.281 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:12.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:12.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.429 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:12.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:12.529 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:12.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.575 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:12.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:12.676 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:12.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.725 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:12.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:12.826 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:12.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.852 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:12.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:12.953 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:12 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 30749/292290 objects degraded (10.520%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:21:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:21:12.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:12 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 30749/292290 objects degraded (10.520%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:21:12.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:21:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.996 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:12.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:12 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 30749/292290 objects degraded (10.520%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:21:13.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:21:13.097 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:13.128 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:13.195 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:13.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:13.268 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:13.296 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:13.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:13.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:13.433 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:13.473 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:13.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:13.574 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:13.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:13.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:13.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:13.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:13.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:13.865 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:13.918 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:13.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:13.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:13 smithi032 ceph-mon[101299]: pgmap v996: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 770 op/s; 30749/290733 objects degraded (10.576%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:13 smithi032 ceph-mon[105748]: pgmap v996: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 770 op/s; 30749/290733 objects degraded (10.576%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:14.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:13 smithi045 ceph-mon[95976]: pgmap v996: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 770 op/s; 30749/290733 objects degraded (10.576%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:14.047 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:14.149 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:14.185 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:14.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:14.286 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.328 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:14.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:14.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:14.474 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:14.575 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:14.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:14.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:14.708 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:14.738 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:14.840 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:14.870 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:14.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:14.972 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:15.013 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:15.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:15.115 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:15.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:15.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:15.258 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:15.299 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:15.401 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:15.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:15.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:15.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:15.594 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:15.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:15.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.739 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:15.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:15.840 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:15.855 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:15 smithi032 ceph-mon[105748]: pgmap v997: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 782 op/s; 30539/285987 objects degraded (10.678%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:21:15.855 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:15 smithi032 ceph-mon[101299]: pgmap v997: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 782 op/s; 30539/285987 objects degraded (10.678%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:21:15.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:15.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:15.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:16.017 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:16.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:16.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:15 smithi045 ceph-mon[95976]: pgmap v997: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 782 op/s; 30539/285987 objects degraded (10.678%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:21:16.119 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:16.158 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:16.260 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:16.314 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:16.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:16.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:16.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:16.558 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:16.585 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:16.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:16.714 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:16.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:16.816 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:16.840 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:16.941 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:16.964 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:17.066 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:17.107 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:17.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.253 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:17.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:17.353 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:17.390 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:17.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:17.492 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:17.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.524 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:17.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:17.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:17.673 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:17.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:17.710 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:21:17.716 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:21:17.776 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:17.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.821 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:17.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:17.922 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[101299]: pgmap v998: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 9.2 MiB/s wr, 802 op/s; 30539/279966 objects degraded (10.908%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:21:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 30539/279966 objects degraded (10.908%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4174194116' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:21:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:21:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3604291523' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[105748]: pgmap v998: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 9.2 MiB/s wr, 802 op/s; 30539/279966 objects degraded (10.908%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:21:17.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 30539/279966 objects degraded (10.908%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:17.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4174194116' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:21:17.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:21:17.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:17 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3604291523' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.995 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:17.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:18.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:17 smithi045 ceph-mon[95976]: pgmap v998: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 9.2 MiB/s wr, 802 op/s; 30539/279966 objects degraded (10.908%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:21:18.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 30539/279966 objects degraded (10.908%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:18.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:17 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4174194116' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:21:18.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:17 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:21:18.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:17 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3604291523' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:18.096 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:18.141 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:18.243 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:18.294 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:18.395 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:18.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.427 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:18.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:18.473 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:21:18.473 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:18.508 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:21:18.508 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:18.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:18.544 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:21:18.544 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:18.565 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:18.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:18.578 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:21:18.578 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:18.604 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:21:18.604 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:18.649 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:21:18.649 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:18.667 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:18.675 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:21:18.675 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:18.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.703 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:18.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:18.703 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:21:18.704 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:18.804 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:18.844 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:18.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:18.946 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3604291523' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[101299]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3720944314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:18.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:21:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:21:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3604291523' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[105748]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T10:21:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3720944314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:18.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:18.980 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:18.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:18 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:21:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:18 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:21:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:18 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3604291523' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:18 smithi045 ceph-mon[95976]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T10:21:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:18 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3720944314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:19.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:19.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:19.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:19.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:19.120 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:19.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:19.221 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:19.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.259 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:19.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:19.360 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:19.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:19.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:19.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:19.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:19.658 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:19.692 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:19.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:19.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:21:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:21:19] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-09-21T10:21:19.793 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:19.836 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:19.890 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 7 1 2024-09-21T10:21:19.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:19.978 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:19 smithi045 ceph-mon[95976]: pgmap v1000: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 6.5 MiB/s wr, 804 op/s; 30539/278397 objects degraded (10.970%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:21:20.080 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:20.121 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:20.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:20.131 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:19 smithi032 ceph-mon[105748]: pgmap v1000: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 6.5 MiB/s wr, 804 op/s; 30539/278397 objects degraded (10.970%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:21:20.131 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:19 smithi032 ceph-mon[101299]: pgmap v1000: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 6.5 MiB/s wr, 804 op/s; 30539/278397 objects degraded (10.970%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:21:20.224 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:20.274 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:20.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:20.376 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:20.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:20.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:20.512 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:20.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:20.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:20.658 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:20.696 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:20.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:20.797 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:20.828 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:20.929 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:20.967 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:21.069 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:21.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.114 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:21.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:21.215 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:21.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.251 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:21.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:21.351 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:21.391 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:21.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:21.493 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:21.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:21.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:21.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.687 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:21.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:21.788 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:21.830 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:21.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:21.968 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:21 smithi045 ceph-mon[95976]: pgmap v1001: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 30270/268665 objects degraded (11.267%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:22.070 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:22.080 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:21 smithi032 ceph-mon[105748]: pgmap v1001: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 30270/268665 objects degraded (11.267%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:22.081 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:21 smithi032 ceph-mon[101299]: pgmap v1001: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 30270/268665 objects degraded (11.267%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:22.102 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:22.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:22.204 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:22.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.234 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:22.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:22.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:22.371 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:22.472 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:22.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:22.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:22.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:22.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.648 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:22.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:22.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:22.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.774 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:22.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:22.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:21:22.875 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:22.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:22.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:22.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 30270/268665 objects degraded (11.267%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:22.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 30270/268665 objects degraded (11.267%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:23.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 30270/268665 objects degraded (11.267%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:23.087 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:23.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:23.189 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:23.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.228 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:23.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:23.329 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:23.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.371 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:23.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:23.472 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.517 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:23.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:23.618 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:23.655 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:23.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:23.757 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:23.794 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:23.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:23.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:21:23.896 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:23.944 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:23.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:23 smithi045 ceph-mon[95976]: pgmap v1002: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 30270/267201 objects degraded (11.329%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:24.046 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:24.061 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:23 smithi032 ceph-mon[101299]: pgmap v1002: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 30270/267201 objects degraded (11.329%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:24.061 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:23 smithi032 ceph-mon[105748]: pgmap v1002: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 30270/267201 objects degraded (11.329%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:21:24.092 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:24.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:24.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:24.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:24.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:24.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:24.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:24.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:24.456 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:24.491 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:24.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:24.593 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:24.628 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:24.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:24.730 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:24.767 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:24.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:24.868 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:24.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:24.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:25.006 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:25.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:25.142 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:25.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:25.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:25.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:25.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:25.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:25.412 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:25.441 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:25.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:25.543 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:25.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.585 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:25.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:25.686 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:25.729 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:25.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:25.831 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:25.875 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:25.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:25.977 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:26.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:26.032 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:25 smithi032 ceph-mon[101299]: pgmap v1003: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 30078/262428 objects degraded (11.461%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:21:26.032 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:25 smithi032 ceph-mon[105748]: pgmap v1003: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 30078/262428 objects degraded (11.461%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:21:26.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:25 smithi045 ceph-mon[95976]: pgmap v1003: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 30078/262428 objects degraded (11.461%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:21:26.072 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:26.092 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:21:26.092 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:21:26.092 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:21:26.092 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:21:26.093 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:21:26.125 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:26.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:26.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:26.255 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.296 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:26.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:26.367 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:26.397 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:26.436 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:26.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:26.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:26.572 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:26.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:26.674 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:26.719 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:26.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:26.821 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:26.867 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:26.969 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:27.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.001 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:27.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:27.003 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:21:27.098 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:26 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 30078/262428 objects degraded (11.461%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:27.098 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:26 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 30078/262428 objects degraded (11.461%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:27.102 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:27.164 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:27.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:27.235 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:27.266 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:26 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 30078/262428 objects degraded (11.461%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:27.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.330 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:27.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:27.431 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:27.472 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:27.574 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:27.619 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:27.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:27.721 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:27.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:27.855 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:27.899 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:27.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:27 smithi032 ceph-mon[105748]: pgmap v1004: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 30078/256506 objects degraded (11.726%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:21:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:27 smithi032 ceph-mon[101299]: pgmap v1004: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 30078/256506 objects degraded (11.726%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:21:28.000 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:28.030 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:28.132 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:28.169 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:28.270 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:27 smithi045 ceph-mon[95976]: pgmap v1004: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 30078/256506 objects degraded (11.726%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:21:28.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:28.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:28.414 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:28.452 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:28.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:28.554 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:28.597 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:28.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:28.698 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:28.731 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:28.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:28.832 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:28.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:28.984 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:29.025 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:29.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:29.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:29.154 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:29.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:29.256 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:29.292 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:29.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:29.394 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:29.428 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:29.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:29.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:29.569 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:29.671 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:29.705 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:29.806 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:29.829 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:21:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:21:29] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-09-21T10:21:29.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.837 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:29.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:29.938 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:29.982 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:30.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:30.122 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:30.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:29 smithi032 ceph-mon[105748]: pgmap v1005: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 30078/255030 objects degraded (11.794%); 2.7 MiB/s, 42 objects/s recovering 2024-09-21T10:21:30.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:29 smithi032 ceph-mon[101299]: pgmap v1005: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 30078/255030 objects degraded (11.794%); 2.7 MiB/s, 42 objects/s recovering 2024-09-21T10:21:30.223 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:30.267 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:29 smithi045 ceph-mon[95976]: pgmap v1005: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 30078/255030 objects degraded (11.794%); 2.7 MiB/s, 42 objects/s recovering 2024-09-21T10:21:30.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:30.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:30.511 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:30.541 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:30.643 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:30.676 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:30.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:30.822 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:30.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:30.924 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:30.962 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:30.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:31.063 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:31.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:31.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:31.210 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.258 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:31.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:31.359 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:31.403 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:31.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:31.505 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:31.541 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:31.642 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:31.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:31.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:31.809 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:31.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:31.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:31.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:31.948 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:21:31.948 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:21:31.948 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:31.948 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:31.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:31.964 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:21:31.964 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:32.002 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:21:32.002 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:32.033 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:21:32.034 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:32.050 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:32.073 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:21:32.073 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:32.087 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:32.099 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:21:32.099 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:32.099 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:31 smithi032 ceph-mon[101299]: pgmap v1006: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 29823/243954 objects degraded (12.225%); 3.7 MiB/s, 59 objects/s recovering 2024-09-21T10:21:32.099 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:31 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 29823/243954 objects degraded (12.225%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:32.100 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:31 smithi032 ceph-mon[105748]: pgmap v1006: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 29823/243954 objects degraded (12.225%); 3.7 MiB/s, 59 objects/s recovering 2024-09-21T10:21:32.100 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:31 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 29823/243954 objects degraded (12.225%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:32.129 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:21:32.129 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:32.189 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:32.232 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:32.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:31 smithi045 ceph-mon[95976]: pgmap v1006: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 29823/243954 objects degraded (12.225%); 3.7 MiB/s, 59 objects/s recovering 2024-09-21T10:21:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:31 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 29823/243954 objects degraded (12.225%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:32.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:32.374 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:32.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:32.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:32.513 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:32.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:32.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:32.659 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:32.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:32.761 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:32.799 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:32.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:32.901 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:32.931 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:32.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:33.032 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:33.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.065 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:33.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:33.150 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:32 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2144292166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:33.151 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:32 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2144292166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:33.165 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:33.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:33.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:32 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2144292166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:33.303 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:33.348 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:33.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:33.419 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:33.450 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:33.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.496 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:33.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:33.597 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:33.636 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:33.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:33.738 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:33.772 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:33.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:33.854 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:21:33.854 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.7 to 1 (10000) 2024-09-21T10:21:33.874 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:33.933 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:34.035 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:34.059 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:34.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:34.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:34.183 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:34.192 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[101299]: pgmap v1007: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 29823/242445 objects degraded (12.301%); 2.3 MiB/s, 37 objects/s recovering 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1656815122' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3439946520' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[105748]: pgmap v1007: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 29823/242445 objects degraded (12.301%); 2.3 MiB/s, 37 objects/s recovering 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1656815122' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3439946520' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:34.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:33 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:34.284 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:33 smithi045 ceph-mon[95976]: pgmap v1007: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 29823/242445 objects degraded (12.301%); 2.3 MiB/s, 37 objects/s recovering 2024-09-21T10:21:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:33 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1656815122' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:21:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:33 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:21:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:33 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3439946520' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:33 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:34.308 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:34.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:34.409 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:34.431 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:34.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:34.532 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:34.555 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:34.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:34.656 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:34.679 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:34.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:34.781 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:34.803 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:34.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:34.905 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:34.968 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:35.070 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:35.092 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:35.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:35.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:35.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:21:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:21:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[101299]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T10:21:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[101299]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T10:21:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:21:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:21:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[105748]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T10:21:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:34 smithi032 ceph-mon[105748]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T10:21:35.216 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:34 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:34 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:21:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:34 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:21:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:34 smithi045 ceph-mon[95976]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T10:21:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:34 smithi045 ceph-mon[95976]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T10:21:35.317 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:35.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:35.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:35.441 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:35.464 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:35.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:35.565 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:35.589 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:35.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:35.691 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:35.713 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:35.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:35.815 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:35.836 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:35.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:35.929 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:21:35.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:35.953 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:35 smithi032 ceph-mon[105748]: pgmap v1009: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 25916/237783 objects degraded (10.899%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:21:35.953 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:35 smithi032 ceph-mon[105748]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T10:21:35.953 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:35 smithi032 ceph-mon[101299]: pgmap v1009: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 25916/237783 objects degraded (10.899%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:21:35.953 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:35 smithi032 ceph-mon[101299]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T10:21:35.980 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:35.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:36.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:36.112 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:36.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:36.251 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:35 smithi045 ceph-mon[95976]: pgmap v1009: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 25916/237783 objects degraded (10.899%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:21:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:35 smithi045 ceph-mon[95976]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T10:21:36.352 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:36.375 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:36.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:36.477 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:36.500 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:36.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:36.601 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:36.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:36.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:36.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:36.748 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:36.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:36.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:36.861 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:21:36.872 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:36.907 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:21:36.974 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:36.998 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:37.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:37.123 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:37.125 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:36 smithi032 ceph-mon[105748]: 8.d starting backfill to osd.7 from (0'0,0'0] MAX to 337'7636 2024-09-21T10:21:37.125 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:36 smithi032 ceph-mon[105748]: 8.c starting backfill to osd.7 from (0'0,0'0] MAX to 337'7693 2024-09-21T10:21:37.125 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:36 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 25916/237783 objects degraded (10.899%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:37.125 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:36 smithi032 ceph-mon[105748]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T10:21:37.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:36 smithi032 ceph-mon[101299]: 8.d starting backfill to osd.7 from (0'0,0'0] MAX to 337'7636 2024-09-21T10:21:37.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:36 smithi032 ceph-mon[101299]: 8.c starting backfill to osd.7 from (0'0,0'0] MAX to 337'7693 2024-09-21T10:21:37.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:36 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 25916/237783 objects degraded (10.899%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:37.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:36 smithi032 ceph-mon[101299]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T10:21:37.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:37.251 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:37.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:36 smithi045 ceph-mon[95976]: 8.d starting backfill to osd.7 from (0'0,0'0] MAX to 337'7636 2024-09-21T10:21:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:36 smithi045 ceph-mon[95976]: 8.c starting backfill to osd.7 from (0'0,0'0] MAX to 337'7693 2024-09-21T10:21:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:36 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 25916/237783 objects degraded (10.899%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:36 smithi045 ceph-mon[95976]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T10:21:37.353 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:37.375 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:37.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:37.479 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:37.502 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:37.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:37.603 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:37.626 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:37.727 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:37.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:37.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:37.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:37.879 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:37.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:37.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:38.008 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:38.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:38.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:38.134 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:38.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:37 smithi032 ceph-mon[101299]: pgmap v1012: 99 pgs: 1 activating, 3 activating+degraded, 12 remapped+peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 139 KiB/s rd, 403 KiB/s wr, 721 op/s; 23706/232785 objects degraded (10.184%); 2216/232785 objects misplaced (0.952%); 3.2 MiB/s, 44 objects/s recovering 2024-09-21T10:21:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:37 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 14 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-09-21T10:21:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:37 smithi032 ceph-mon[101299]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T10:21:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:37 smithi032 ceph-mon[105748]: pgmap v1012: 99 pgs: 1 activating, 3 activating+degraded, 12 remapped+peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 139 KiB/s rd, 403 KiB/s wr, 721 op/s; 23706/232785 objects degraded (10.184%); 2216/232785 objects misplaced (0.952%); 3.2 MiB/s, 44 objects/s recovering 2024-09-21T10:21:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:37 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 14 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-09-21T10:21:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:37 smithi032 ceph-mon[105748]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T10:21:38.236 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:38.262 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:37 smithi045 ceph-mon[95976]: pgmap v1012: 99 pgs: 1 activating, 3 activating+degraded, 12 remapped+peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 139 KiB/s rd, 403 KiB/s wr, 721 op/s; 23706/232785 objects degraded (10.184%); 2216/232785 objects misplaced (0.952%); 3.2 MiB/s, 44 objects/s recovering 2024-09-21T10:21:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:37 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 14 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-09-21T10:21:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:37 smithi045 ceph-mon[95976]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T10:21:38.364 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:38.387 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:38.488 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:38.515 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:38.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:38.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:38.639 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:38.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:38.741 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:38.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:38.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:38.866 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:38.891 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:38.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:38.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:39.017 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:39.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:39.119 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:39.143 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:39.244 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:39.271 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:39.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:39.373 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:39.400 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:39.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:39.525 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:39.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:39.626 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:39.651 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:39.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:39.753 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:39.774 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:39.863 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:21:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:21:39] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.51.0" 2024-09-21T10:21:39.876 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:39.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:39.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:40.006 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:40.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.030 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:40.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:40.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:40.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:40.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:39 smithi032 ceph-mon[105748]: pgmap v1015: 99 pgs: 1 activating, 3 activating+degraded, 12 remapped+peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 208 KiB/s rd, 604 KiB/s wr, 630 op/s; 23706/232032 objects degraded (10.217%); 2216/232032 objects misplaced (0.955%); 886 KiB/s, 4 objects/s recovering 2024-09-21T10:21:40.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:39 smithi032 ceph-mon[101299]: pgmap v1015: 99 pgs: 1 activating, 3 activating+degraded, 12 remapped+peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 208 KiB/s rd, 604 KiB/s wr, 630 op/s; 23706/232032 objects degraded (10.217%); 2216/232032 objects misplaced (0.955%); 886 KiB/s, 4 objects/s recovering 2024-09-21T10:21:40.255 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:40.280 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:40.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:39 smithi045 ceph-mon[95976]: pgmap v1015: 99 pgs: 1 activating, 3 activating+degraded, 12 remapped+peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 208 KiB/s rd, 604 KiB/s wr, 630 op/s; 23706/232032 objects degraded (10.217%); 2216/232032 objects misplaced (0.955%); 886 KiB/s, 4 objects/s recovering 2024-09-21T10:21:40.382 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:40.404 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:40.506 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:40.514 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:21:40.514 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:21:40.528 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:40.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:40.630 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:40.653 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:40.723 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:40.754 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:40.777 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:40.782 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.799 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:40.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:40.899 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:40.934 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:41.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:41.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:41.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:41.174 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:41.178 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:40 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 14 pgs inactive, 11 pgs peering) 2024-09-21T10:21:41.179 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:40 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 14 pgs inactive, 11 pgs peering) 2024-09-21T10:21:41.201 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:41.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:40 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 14 pgs inactive, 11 pgs peering) 2024-09-21T10:21:41.303 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:41.331 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:41.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:41.432 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:41.460 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:41.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:41.562 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:41.590 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:41.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:41.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:41.722 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:41.824 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:41.856 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:41.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:41.983 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:41.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:42.029 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:41 smithi032 ceph-mon[101299]: pgmap v1016: 99 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 147 KiB/s rd, 427 KiB/s wr, 471 op/s; 28739/231600 objects degraded (12.409%); 9586/231600 objects misplaced (4.139%); 10 MiB/s, 153 objects/s recovering 2024-09-21T10:21:42.029 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:41 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 28739/231600 objects degraded (12.409%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:42.029 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:41 smithi032 ceph-mon[105748]: pgmap v1016: 99 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 147 KiB/s rd, 427 KiB/s wr, 471 op/s; 28739/231600 objects degraded (12.409%); 9586/231600 objects misplaced (4.139%); 10 MiB/s, 153 objects/s recovering 2024-09-21T10:21:42.029 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:41 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 28739/231600 objects degraded (12.409%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:42.085 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:42.108 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:42.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:42.210 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:42.233 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:41 smithi045 ceph-mon[95976]: pgmap v1016: 99 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 147 KiB/s rd, 427 KiB/s wr, 471 op/s; 28739/231600 objects degraded (12.409%); 9586/231600 objects misplaced (4.139%); 10 MiB/s, 153 objects/s recovering 2024-09-21T10:21:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:41 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 28739/231600 objects degraded (12.409%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:42.335 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:42.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:42.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:42.461 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:42.492 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:42.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:42.594 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:42.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:42.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:42.753 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:42.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:42.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:42.877 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:42.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:42.979 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:43.002 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:43.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:43.104 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:43.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:43.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:43.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:43.253 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:43.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:43.378 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:43.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:43.480 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:43.505 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:43.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:43.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:43.634 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:43.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:43.736 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:43.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:43.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:43.865 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:43.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:43.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:43.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:44.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:44.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:44.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:44.150 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:44.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:43 smithi032 ceph-mon[101299]: pgmap v1017: 99 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 28739/231417 objects degraded (12.419%); 9586/231417 objects misplaced (4.142%); 7.9 MiB/s, 128 objects/s recovering 2024-09-21T10:21:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:43 smithi032 ceph-mon[105748]: pgmap v1017: 99 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 28739/231417 objects degraded (12.419%); 9586/231417 objects misplaced (4.142%); 7.9 MiB/s, 128 objects/s recovering 2024-09-21T10:21:44.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:44.279 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:44.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:43 smithi045 ceph-mon[95976]: pgmap v1017: 99 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 28739/231417 objects degraded (12.419%); 9586/231417 objects misplaced (4.142%); 7.9 MiB/s, 128 objects/s recovering 2024-09-21T10:21:44.381 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:44.406 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:44.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:44.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:44.535 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:44.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:44.637 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:44.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:44.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:44.763 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:44.793 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:44.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:44.895 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:44.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:45.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:45.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:45.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:45.140 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:21:45.149 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:45.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:45.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:45.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:45.306 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:45.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:45.408 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:45.434 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:45.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:45.536 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:45.563 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:45.665 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:45.690 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:45.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:45.815 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:45.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:45.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:45.945 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:45.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:46.047 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:46.076 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:46.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:46.178 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:46.206 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:46.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:45 smithi032 ceph-mon[101299]: pgmap v1018: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 26742/230118 objects degraded (11.621%); 6908/230118 objects misplaced (3.002%); 11 MiB/s, 173 objects/s recovering 2024-09-21T10:21:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:45 smithi032 ceph-mon[105748]: pgmap v1018: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 26742/230118 objects degraded (11.621%); 6908/230118 objects misplaced (3.002%); 11 MiB/s, 173 objects/s recovering 2024-09-21T10:21:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:45 smithi045 ceph-mon[95976]: pgmap v1018: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 26742/230118 objects degraded (11.621%); 6908/230118 objects misplaced (3.002%); 11 MiB/s, 173 objects/s recovering 2024-09-21T10:21:46.307 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:46.331 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:46.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:46.458 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:46.560 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:46.590 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:46.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:46.719 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:46.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:46.820 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:46.849 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:46.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:46.951 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:46.981 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:46.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:47.083 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:47.111 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:46 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 26742/230118 objects degraded (11.621%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:46 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 26742/230118 objects degraded (11.621%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:47.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:47.243 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:47.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:46 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 26742/230118 objects degraded (11.621%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:47.345 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:47.370 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:47.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:47.472 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:47.494 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:47.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:47.596 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:47.620 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:47.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:47.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:47.752 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:47.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:47.853 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:47.883 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:47.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:47.952 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:21:47.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:48.014 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:48.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:48.116 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:48.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:48.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:47 smithi032 ceph-mon[101299]: pgmap v1019: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 26742/227769 objects degraded (11.741%); 6908/227769 objects misplaced (3.033%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:21:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:47 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2786506812' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:47 smithi032 ceph-mon[105748]: pgmap v1019: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 26742/227769 objects degraded (11.741%); 6908/227769 objects misplaced (3.033%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:21:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:47 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2786506812' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:48.249 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:48.278 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:48.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:47 smithi045 ceph-mon[95976]: pgmap v1019: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 26742/227769 objects degraded (11.741%); 6908/227769 objects misplaced (3.033%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:21:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:47 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2786506812' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:48.379 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:48.404 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:48.506 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:48.529 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:48.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:48.631 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:48.661 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:48.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:48.762 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:48.793 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:48.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:48.922 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:49.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:49.046 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:49.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2786506812' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[105748]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2786506812' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[101299]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:49.122 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:49.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:49.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:49.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:48 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:48 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2786506812' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:48 smithi045 ceph-mon[95976]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T10:21:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:49.295 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:49.397 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:49.425 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:49.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:49.527 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:49.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:49.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:49.666 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:49.696 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:49.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:49.797 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:49.823 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:49.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:49.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:49.953 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:49.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:49.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:21:49] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.51.0" 2024-09-21T10:21:49.967 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:21:50.055 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:50.080 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:50.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:50.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:50.206 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:50.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:49 smithi045 ceph-mon[95976]: pgmap v1021: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 26742/227205 objects degraded (11.770%); 6908/227205 objects misplaced (3.040%); 8.6 MiB/s, 139 objects/s recovering 2024-09-21T10:21:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:49 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/760705705' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:21:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:49 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:21:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:50.308 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:50.330 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:50.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:50.432 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:50.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[105748]: pgmap v1021: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 26742/227205 objects degraded (11.770%); 6908/227205 objects misplaced (3.040%); 8.6 MiB/s, 139 objects/s recovering 2024-09-21T10:21:50.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/760705705' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:21:50.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:21:50.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:50.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[101299]: pgmap v1021: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 26742/227205 objects degraded (11.770%); 6908/227205 objects misplaced (3.040%); 8.6 MiB/s, 139 objects/s recovering 2024-09-21T10:21:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/760705705' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:21:50.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:21:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:50.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:50.459 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:50.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:50.561 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:50.584 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:50.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:50.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:50.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.713 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:50.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:50.808 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 0 1 2024-09-21T10:21:50.814 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:50.877 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:50.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:50.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:50.985 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:51.008 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:51.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:51.133 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:21:51.133 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[105748]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T10:21:51.133 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:21:51.133 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:21:51.133 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:51.133 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:21:51.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[101299]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T10:21:51.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:21:51.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:21:51.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:50 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:51.135 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:51.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:51.236 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:51.265 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:51.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:50 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:21:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:50 smithi045 ceph-mon[95976]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T10:21:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:21:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:21:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:50 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:21:51.367 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:51.396 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:51.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:51.497 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:51.521 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:51.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:51.623 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:51.634 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:21:51.634 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:51.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:51.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:51.655 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:21:51.655 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:51.675 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:21:51.675 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:51.698 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:21:51.698 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:51.719 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:21:51.719 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:51.742 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:21:51.780 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:51.781 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:21:51.781 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:51.781 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:51.801 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:21:51.802 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:51.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.816 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:51.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:51.917 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:51.946 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:52.048 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:52.071 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:52.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:52.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:21:52.173 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:52.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:52.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:51 smithi032 ceph-mon[101299]: pgmap v1023: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 26104/223185 objects degraded (11.696%); 6723/223185 objects misplaced (3.012%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:21:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:51 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2177566362' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:51 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 26104/223185 objects degraded (11.696%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:51 smithi032 ceph-mon[105748]: pgmap v1023: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 26104/223185 objects degraded (11.696%); 6723/223185 objects misplaced (3.012%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:21:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:51 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2177566362' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:51 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 26104/223185 objects degraded (11.696%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:51 smithi045 ceph-mon[95976]: pgmap v1023: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 26104/223185 objects degraded (11.696%); 6723/223185 objects misplaced (3.012%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:21:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:51 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2177566362' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:51 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 26104/223185 objects degraded (11.696%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:52.297 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:52.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:52.422 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:52.446 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:52.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:52.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:52.571 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:52.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:52.673 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:52.701 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:52.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:52.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:52.827 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:52.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:52.929 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:52.957 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:52.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:52.965 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:53.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:53.084 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:53.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:53.186 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:53.208 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:53.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:53.335 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:53.437 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:53.464 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:53.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:53.542 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:53.565 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:53.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:53.695 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:53.722 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:53.824 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:53.853 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:53.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:53.955 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:53.980 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:54.064 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:53 smithi032 ceph-mon[105748]: pgmap v1024: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 312 op/s; 26104/222615 objects degraded (11.726%); 6723/222615 objects misplaced (3.020%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:21:54.064 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:53 smithi032 ceph-mon[101299]: pgmap v1024: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 312 op/s; 26104/222615 objects degraded (11.726%); 6723/222615 objects misplaced (3.020%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:21:54.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:54.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:54.210 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:54.235 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:53 smithi045 ceph-mon[95976]: pgmap v1024: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 312 op/s; 26104/222615 objects degraded (11.726%); 6723/222615 objects misplaced (3.020%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:21:54.337 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:54.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.363 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:54.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:54.464 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:54.488 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:54.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:54.590 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:54.617 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:54.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:54.719 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:54.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:54.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:54.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:54.869 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:54.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:54.948 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:21:54.971 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:54.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:54.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:55.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:55.122 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:55.224 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:55.246 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:55.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:55.348 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:55.374 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:55.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:55.475 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:55.504 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:55.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:55.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:55.632 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:55.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:55.757 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:55.859 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:55.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:55.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:55.986 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:56.008 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:56.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:56.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:56.136 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:56.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:55 smithi032 ceph-mon[105748]: pgmap v1025: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 274 op/s; 25390/221187 objects degraded (11.479%); 6556/221187 objects misplaced (2.964%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:21:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:55 smithi032 ceph-mon[101299]: pgmap v1025: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 274 op/s; 25390/221187 objects degraded (11.479%); 6556/221187 objects misplaced (2.964%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:21:56.237 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:56.263 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:56.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:55 smithi045 ceph-mon[95976]: pgmap v1025: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 274 op/s; 25390/221187 objects degraded (11.479%); 6556/221187 objects misplaced (2.964%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:21:56.365 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:56.390 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:56.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:56.492 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:56.519 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:56.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:56.621 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:56.645 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:56.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:56.747 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:56.775 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:56.877 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:56.900 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:56.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:57.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:57.030 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:57.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:57.132 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:57.159 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:57.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:57.260 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:56 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 25390/221187 objects degraded (11.479%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:57.286 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:57.334 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:56 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 25390/221187 objects degraded (11.479%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:57.334 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:56 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 25390/221187 objects degraded (11.479%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:21:57.338 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:21:57.338 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:21:57.338 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.338 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:127471 8.a 8:5374f573:::benchmark_data_smithi032_233595_object27626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-09-21T10:21:57.338 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:21:57.333095+0000", 2024-09-21T10:21:57.338 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.002711869, 2024-09-21T10:21:57.338 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0027407640000000001, 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 127471 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:21:57.333095+0000", 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:21:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:21:57.333095+0000", 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:21:57.333098+0000", 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.39e-06 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:21:57.333105+0000", 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7139999999999993e-06 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.340 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:21:57.333124+0000", 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8107000000000001e-05 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:21:57.333134+0000", 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0200000000000001e-05 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:21:57.333209+0000", 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5409999999999998e-05 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:21:57.341 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:21:57.333402+0000", 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000192692 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:21:57.333589+0000", 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000186959 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 1 2024-09-21T10:21:57.342 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:21:57.388 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:57.411 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:57.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:57.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:57.537 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:57.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:57.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:57.668 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:57.770 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:57.800 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:57.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:57.901 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:57.938 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:57.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:58.040 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:58.066 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:58.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:58.168 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:58.191 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:58.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:57 smithi045 ceph-mon[95976]: pgmap v1026: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 25390/219354 objects degraded (11.575%); 6556/219354 objects misplaced (2.989%); 9.9 MiB/s, 157 objects/s recovering 2024-09-21T10:21:58.293 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:58.294 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:57 smithi032 ceph-mon[101299]: pgmap v1026: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 25390/219354 objects degraded (11.575%); 6556/219354 objects misplaced (2.989%); 9.9 MiB/s, 157 objects/s recovering 2024-09-21T10:21:58.294 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:57 smithi032 ceph-mon[105748]: pgmap v1026: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 25390/219354 objects degraded (11.575%); 6556/219354 objects misplaced (2.989%); 9.9 MiB/s, 157 objects/s recovering 2024-09-21T10:21:58.317 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:58.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:58.419 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:58.445 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:58.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:58.547 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:58.569 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:58.671 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:58.684 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:21:58.699 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:58.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:58.710 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:58.801 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:58.829 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:58.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:58.931 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:58.954 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:58.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:58.990 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:21:59.055 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:59.081 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:59.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:59.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:59.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:59.285 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:58 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2546415173' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:59.285 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:58 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2546415173' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:59.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:58 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2546415173' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:59.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:59.336 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:59.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:59.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:59.464 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:59.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:59.566 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:59.593 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:59.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:59.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:59.726 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:21:59.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:59.828 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:59.852 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:59.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:59.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:59.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:21:59] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.51.0" 2024-09-21T10:21:59.978 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:21:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:59.997 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.0 to 1 (10000) 2024-09-21T10:22:00.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:00.102 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:00.204 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:00.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:59 smithi045 ceph-mon[95976]: pgmap v1027: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 277 op/s; 25390/218880 objects degraded (11.600%); 6556/218880 objects misplaced (2.995%); 8.4 MiB/s, 134 objects/s recovering 2024-09-21T10:22:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:59 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2546415173' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:59 smithi045 ceph-mon[95976]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T10:22:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/205357357' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:21:59 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:00.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:00.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[101299]: pgmap v1027: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 277 op/s; 25390/218880 objects degraded (11.600%); 6556/218880 objects misplaced (2.995%); 8.4 MiB/s, 134 objects/s recovering 2024-09-21T10:22:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2546415173' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[101299]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T10:22:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/205357357' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:00.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[105748]: pgmap v1027: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 277 op/s; 25390/218880 objects degraded (11.600%); 6556/218880 objects misplaced (2.995%); 8.4 MiB/s, 134 objects/s recovering 2024-09-21T10:22:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2546415173' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[105748]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T10:22:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/205357357' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:21:59 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:00.458 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:00.481 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:00.583 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:00.608 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:00.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:00.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:00.733 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:00.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:00.835 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:00.858 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:00.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:00.959 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:00.983 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:00.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:01.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:01.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:01.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:22:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:01 smithi032 ceph-mon[101299]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T10:22:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:22:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:01 smithi032 ceph-mon[105748]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T10:22:01.210 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:01.233 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:01.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:01.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:22:01.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:00 smithi045 ceph-mon[95976]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T10:22:01.335 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:01.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.359 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:01.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:01.460 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:01.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:01.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:01.589 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:01.616 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:01.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:01.717 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:01.743 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:01.845 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:01.871 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:01.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:01.972 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:02.000 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:02.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:02.102 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:02.126 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:02.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[105748]: pgmap v1030: 99 pgs: 1 peering, 2 remapped+peering, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 24586/215178 objects degraded (11.426%); 6383/215178 objects misplaced (2.966%); 12 MiB/s, 189 objects/s recovering 2024-09-21T10:22:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[105748]: 8.c starting backfill to osd.0 from (0'0,0'0] MAX to 345'8057 2024-09-21T10:22:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[105748]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T10:22:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[105748]: 8.c continuing backfill to osd.7 from (314'1200,345'8057] MIN to 345'8057 2024-09-21T10:22:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 24586/215178 objects degraded (11.426%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[101299]: pgmap v1030: 99 pgs: 1 peering, 2 remapped+peering, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 24586/215178 objects degraded (11.426%); 6383/215178 objects misplaced (2.966%); 12 MiB/s, 189 objects/s recovering 2024-09-21T10:22:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[101299]: 8.c starting backfill to osd.0 from (0'0,0'0] MAX to 345'8057 2024-09-21T10:22:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[101299]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T10:22:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[101299]: 8.c continuing backfill to osd.7 from (314'1200,345'8057] MIN to 345'8057 2024-09-21T10:22:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:02 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 24586/215178 objects degraded (11.426%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:02.227 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:02.269 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:02 smithi045 ceph-mon[95976]: pgmap v1030: 99 pgs: 1 peering, 2 remapped+peering, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 24586/215178 objects degraded (11.426%); 6383/215178 objects misplaced (2.966%); 12 MiB/s, 189 objects/s recovering 2024-09-21T10:22:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:02 smithi045 ceph-mon[95976]: 8.c starting backfill to osd.0 from (0'0,0'0] MAX to 345'8057 2024-09-21T10:22:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:02 smithi045 ceph-mon[95976]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T10:22:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:02 smithi045 ceph-mon[95976]: 8.c continuing backfill to osd.7 from (314'1200,345'8057] MIN to 345'8057 2024-09-21T10:22:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:02 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 24586/215178 objects degraded (11.426%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:02.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:02.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:02.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:02.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:02.545 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:02.647 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:02.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:02.782 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:02.809 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:02.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:02.911 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.941 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:02.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:03.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.079 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:03.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:03.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:03.213 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:03 smithi045 ceph-mon[95976]: 8.9 starting backfill to osd.0 from (0'0,0'0] MAX to 345'8015 2024-09-21T10:22:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:03 smithi045 ceph-mon[95976]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T10:22:03.316 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:03.344 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:03.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:03.445 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:03 smithi032 ceph-mon[105748]: 8.9 starting backfill to osd.0 from (0'0,0'0] MAX to 345'8015 2024-09-21T10:22:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:03 smithi032 ceph-mon[105748]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T10:22:03.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:03 smithi032 ceph-mon[101299]: 8.9 starting backfill to osd.0 from (0'0,0'0] MAX to 345'8015 2024-09-21T10:22:03.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:03 smithi032 ceph-mon[101299]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T10:22:03.470 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:03.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:03.572 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:03.599 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:03.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:03.701 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:03.729 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:03.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:03.831 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:03.853 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:03.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:03.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:04.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:04.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:04.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:04.232 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:04.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:04 smithi045 ceph-mon[95976]: pgmap v1033: 99 pgs: 1 peering, 2 remapped+peering, 1 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 24589/214734 objects degraded (11.451%); 6383/214734 objects misplaced (2.973%); 13 MiB/s, 200 objects/s recovering 2024-09-21T10:22:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:04 smithi045 ceph-mon[95976]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T10:22:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:04.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:04.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:04.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:04 smithi032 ceph-mon[101299]: pgmap v1033: 99 pgs: 1 peering, 2 remapped+peering, 1 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 24589/214734 objects degraded (11.451%); 6383/214734 objects misplaced (2.973%); 13 MiB/s, 200 objects/s recovering 2024-09-21T10:22:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:04 smithi032 ceph-mon[101299]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T10:22:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:04.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:04 smithi032 ceph-mon[105748]: pgmap v1033: 99 pgs: 1 peering, 2 remapped+peering, 1 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 24589/214734 objects degraded (11.451%); 6383/214734 objects misplaced (2.973%); 13 MiB/s, 200 objects/s recovering 2024-09-21T10:22:04.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:04 smithi032 ceph-mon[105748]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T10:22:04.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:04.461 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:04.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:04.589 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:04.619 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:04.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:04.753 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:04.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:04.879 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:04.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:04.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:05.003 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:05.104 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:05.133 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:05.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:05.235 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:05.261 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:05 smithi045 ceph-mon[95976]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T10:22:05.318 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:05 smithi032 ceph-mon[105748]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T10:22:05.318 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:05 smithi032 ceph-mon[101299]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T10:22:05.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:05.388 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:05.489 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:05.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:05.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:05.613 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:05.638 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:05.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:05.740 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:05.768 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:05.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:05.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:05.891 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:05.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:06.016 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:06.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:06.117 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:06.146 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:06.248 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:06.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:06 smithi045 ceph-mon[95976]: pgmap v1036: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 peering, 2 remapped+peering, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 24003/213333 objects degraded (11.251%); 7420/213333 objects misplaced (3.478%); 5.7 MiB/s, 146 objects/s recovering 2024-09-21T10:22:06.376 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:06.411 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:06.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:06.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:06 smithi032 ceph-mon[105748]: pgmap v1036: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 peering, 2 remapped+peering, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 24003/213333 objects degraded (11.251%); 7420/213333 objects misplaced (3.478%); 5.7 MiB/s, 146 objects/s recovering 2024-09-21T10:22:06.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:06 smithi032 ceph-mon[101299]: pgmap v1036: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 peering, 2 remapped+peering, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 24003/213333 objects degraded (11.251%); 7420/213333 objects misplaced (3.478%); 5.7 MiB/s, 146 objects/s recovering 2024-09-21T10:22:06.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:06.538 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:06.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:06.640 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:06.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.672 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:06.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:06.774 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:06.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:06.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:06.900 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:06.927 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:06.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:07.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:07.051 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:07.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:07.153 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:07.176 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:07.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:07.260 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:07.278 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:07 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 24003/213333 objects degraded (11.251%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:07.308 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:07.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:22:07.376 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:07 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 24003/213333 objects degraded (11.251%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:07.376 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:07 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 24003/213333 objects degraded (11.251%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:07.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:07.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:07.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:07.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:07.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:07.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:07.692 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:07.794 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:07.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.823 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:07.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:07.924 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:07.953 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:07.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:08.055 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:08.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.089 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:08.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:08.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:22:08.190 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:08.218 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:08 smithi045 ceph-mon[95976]: pgmap v1037: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 21967/211035 objects degraded (10.409%); 13268/211035 objects misplaced (6.287%); 9.1 MiB/s, 187 objects/s recovering 2024-09-21T10:22:08.321 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:08.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:08.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:08.412 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:08 smithi032 ceph-mon[105748]: pgmap v1037: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 21967/211035 objects degraded (10.409%); 13268/211035 objects misplaced (6.287%); 9.1 MiB/s, 187 objects/s recovering 2024-09-21T10:22:08.413 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:08 smithi032 ceph-mon[101299]: pgmap v1037: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 21967/211035 objects degraded (10.409%); 13268/211035 objects misplaced (6.287%); 9.1 MiB/s, 187 objects/s recovering 2024-09-21T10:22:08.448 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:08.474 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:08.576 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:08.604 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:08.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:08.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:08.728 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:08.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:08.854 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:08.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:08.955 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:08.981 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:08.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:09.042 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:22:09.042 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:22:09.066 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:22:09.067 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:22:09.083 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:09.089 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:22:09.089 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:22:09.109 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:09.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:09.111 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:22:09.111 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:22:09.129 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:22:09.129 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:22:09.147 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:22:09.147 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:22:09.165 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:22:09.165 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:22:09.182 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:22:09.183 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:22:09.211 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:09.238 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:09.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4037731623' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:09.340 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:09.368 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:09.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4037731623' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4037731623' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:09.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:09.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:09.600 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:09.623 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:09.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:09.724 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:09.735 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:09.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:09.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:09.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:09.859 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:22:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:22:09] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.51.0" 2024-09-21T10:22:09.875 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:09.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:09.977 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:09.999 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:09.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:10.101 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:10.125 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:10.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:10 smithi032 ceph-mon[105748]: pgmap v1038: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 21967/210408 objects degraded (10.440%); 13268/210408 objects misplaced (6.306%); 7.7 MiB/s, 158 objects/s recovering 2024-09-21T10:22:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:10 smithi032 ceph-mon[101299]: pgmap v1038: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 21967/210408 objects degraded (10.440%); 13268/210408 objects misplaced (6.306%); 7.7 MiB/s, 158 objects/s recovering 2024-09-21T10:22:10.227 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:10.259 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:10.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:10.360 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:10.383 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:10.485 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:10.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:10 smithi045 ceph-mon[95976]: pgmap v1038: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 21967/210408 objects degraded (10.440%); 13268/210408 objects misplaced (6.306%); 7.7 MiB/s, 158 objects/s recovering 2024-09-21T10:22:10.613 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:10.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:10.742 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:10.769 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:10.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:10.800 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:10.870 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:10.895 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:10.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:11.021 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:11.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:11.123 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:11.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:11.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:11.281 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:11.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:11.382 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:11.407 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:11.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:11.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:11.535 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:11.636 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:11.663 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:11.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:11.765 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:11.793 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:11.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:11.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:11.921 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:11.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:12.023 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:12.054 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:12.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:12.155 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:12.162 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:12 smithi032 ceph-mon[105748]: pgmap v1039: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 21668/206442 objects degraded (10.496%); 12548/206442 objects misplaced (6.078%); 12 MiB/s, 219 objects/s recovering 2024-09-21T10:22:12.162 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:12 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 21668/206442 objects degraded (10.496%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:12.162 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:12 smithi032 ceph-mon[101299]: pgmap v1039: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 21668/206442 objects degraded (10.496%); 12548/206442 objects misplaced (6.078%); 12 MiB/s, 219 objects/s recovering 2024-09-21T10:22:12.162 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:12 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 21668/206442 objects degraded (10.496%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:12.183 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:12.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:12.285 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:12.311 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:12.343 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:22:12.413 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:12.437 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:12.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:12 smithi045 ceph-mon[95976]: pgmap v1039: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 21668/206442 objects degraded (10.496%); 12548/206442 objects misplaced (6.078%); 12 MiB/s, 219 objects/s recovering 2024-09-21T10:22:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:12 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 21668/206442 objects degraded (10.496%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:12.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:12.566 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:12.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:12.668 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:12.696 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:12.798 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:12.823 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:12.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:12.924 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:12.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:12.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:13.052 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:13.082 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:13.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:13.184 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:13.219 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:13.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:13.320 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:13.354 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:13.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:13.456 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:13.482 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:13.585 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:13.610 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:13.712 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:13.738 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:13.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:13.840 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:13.863 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:13.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:13.939 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:22:13.939 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:22:13.965 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:13.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:13.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:14.099 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:14.126 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:14.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:14.179 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:14 smithi032 ceph-mon[105748]: pgmap v1040: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 21668/205923 objects degraded (10.522%); 12548/205923 objects misplaced (6.094%); 10 MiB/s, 184 objects/s recovering 2024-09-21T10:22:14.180 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:14 smithi032 ceph-mon[101299]: pgmap v1040: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 21668/205923 objects degraded (10.522%); 12548/205923 objects misplaced (6.094%); 10 MiB/s, 184 objects/s recovering 2024-09-21T10:22:14.227 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:14.253 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:14.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:14.355 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:14.380 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:14.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:14.482 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:14.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:14.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:14 smithi045 ceph-mon[95976]: pgmap v1040: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 21668/205923 objects degraded (10.522%); 12548/205923 objects misplaced (6.094%); 10 MiB/s, 184 objects/s recovering 2024-09-21T10:22:14.615 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:14.645 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:14.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:14.747 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:14.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:14.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:14.877 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:14.900 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:15.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:15.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:15.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:15.049 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:22:15.119 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:22:15.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:15.190 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:15.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:15.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.330 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:15.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:15.398 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:15 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3535868730' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:22:15.399 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:15 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:22:15.399 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:15 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3535868730' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:22:15.399 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:15 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:22:15.431 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:15.454 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:15 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3535868730' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:22:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:15 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:22:15.555 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:15.578 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:15.679 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:15.708 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:15.810 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:15.835 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:15.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:15.890 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:22:15.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:22:15.937 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:15.963 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:16.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.094 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:16.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:16.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:16.196 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:16 smithi032 ceph-mon[105748]: pgmap v1041: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 21570/204456 objects degraded (10.550%); 12450/204456 objects misplaced (6.089%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:22:16.197 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:16 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:22:16.197 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:16 smithi032 ceph-mon[105748]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T10:22:16.197 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:16 smithi032 ceph-mon[101299]: pgmap v1041: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 21570/204456 objects degraded (10.550%); 12450/204456 objects misplaced (6.089%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:22:16.197 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:16 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:22:16.197 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:16 smithi032 ceph-mon[101299]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T10:22:16.220 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:16.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:16.322 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:16.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:16.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:16 smithi045 ceph-mon[95976]: pgmap v1041: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 21570/204456 objects degraded (10.550%); 12450/204456 objects misplaced (6.089%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:22:16.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:16 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:22:16.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:16 smithi045 ceph-mon[95976]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T10:22:16.448 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:16.482 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:16.509 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:16.512 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:16.571 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:16.584 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:16.625 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:16.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:16.727 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:16.759 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:16.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:16.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:16.889 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:16.991 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:17.014 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:17.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:17.057 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:22:17.116 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:17.146 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:17.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:17.247 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:17.275 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:17.277 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:17 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3171780591' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:17.277 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:17 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:17.277 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 21570/204456 objects degraded (10.550%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:17.277 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:17 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3171780591' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:17.277 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:17 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:17.277 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 21570/204456 objects degraded (10.550%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:17.377 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:17.399 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:17.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:17.501 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:17 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3171780591' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:17 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 21570/204456 objects degraded (10.550%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:17.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:17.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:17.639 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:17.675 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:17.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:17.802 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:17.904 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:17.933 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:17.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:18.034 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:18.064 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:18.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:18.166 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:18.189 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:18.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:18.291 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:18.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:18.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:18.348 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:18 smithi032 ceph-mon[101299]: pgmap v1043: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 313 op/s; 21366/201630 objects degraded (10.597%); 11796/201630 objects misplaced (5.850%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:22:18.348 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:18 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:18.348 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:18 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:18.348 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:18 smithi032 ceph-mon[101299]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T10:22:18.349 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:18 smithi032 ceph-mon[105748]: pgmap v1043: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 313 op/s; 21366/201630 objects degraded (10.597%); 11796/201630 objects misplaced (5.850%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:22:18.349 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:18 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:18.349 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:18 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:18.349 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:18 smithi032 ceph-mon[105748]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T10:22:18.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:18.448 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:18.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:18 smithi045 ceph-mon[95976]: pgmap v1043: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 313 op/s; 21366/201630 objects degraded (10.597%); 11796/201630 objects misplaced (5.850%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:22:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:18 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:18 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:18 smithi045 ceph-mon[95976]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T10:22:18.550 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:18.576 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:18.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:18.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:18.703 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:18.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:18.804 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:18.828 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:18.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:18.930 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:18.954 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:19.056 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:19.083 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:19.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:19.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:19.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:19.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:19.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:19.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:19.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:19.439 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:19.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:19.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:19.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:19.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:19.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:19.672 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:22:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:22:19] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-21T10:22:19.673 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:19 smithi032 ceph-mon[101299]: pgmap v1045: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 227 op/s; 21366/200973 objects degraded (10.631%); 11796/200973 objects misplaced (5.869%); 5.9 MiB/s, 93 objects/s recovering 2024-09-21T10:22:19.673 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:19.673 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:19.673 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:19 smithi032 ceph-mon[105748]: pgmap v1045: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 227 op/s; 21366/200973 objects degraded (10.631%); 11796/200973 objects misplaced (5.869%); 5.9 MiB/s, 93 objects/s recovering 2024-09-21T10:22:19.673 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:19.673 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:19.706 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:19.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:19 smithi045 ceph-mon[95976]: pgmap v1045: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 227 op/s; 21366/200973 objects degraded (10.631%); 11796/200973 objects misplaced (5.869%); 5.9 MiB/s, 93 objects/s recovering 2024-09-21T10:22:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:19.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:19.871 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:19.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:19.973 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:19.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:20.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:20.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:20.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:20.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:20.261 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:20.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:20.363 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:20.387 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:20.489 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:20.516 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:20.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:20.618 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:20.646 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:20.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:20.747 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:20.770 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:20.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:20.872 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:20.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:20.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:21.000 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:21.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:21.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:21.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:21.150 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:21.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:21.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:21.281 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:21.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:21.382 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:21.410 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:21.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:21.512 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:21.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:21.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:21.642 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:21.670 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:21.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:21.689 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:21 smithi032 ceph-mon[105748]: pgmap v1046: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 21041/196368 objects degraded (10.715%); 10969/196368 objects misplaced (5.586%); 12 MiB/s, 197 objects/s recovering 2024-09-21T10:22:21.689 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:21 smithi032 ceph-mon[101299]: pgmap v1046: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 21041/196368 objects degraded (10.715%); 10969/196368 objects misplaced (5.586%); 12 MiB/s, 197 objects/s recovering 2024-09-21T10:22:21.773 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:21 smithi045 ceph-mon[95976]: pgmap v1046: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 21041/196368 objects degraded (10.715%); 10969/196368 objects misplaced (5.586%); 12 MiB/s, 197 objects/s recovering 2024-09-21T10:22:21.801 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:21.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:21.904 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:21.932 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:21.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:21.950 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:22:21.952 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:22:22.033 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:22.060 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:22.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:22.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:22.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:22.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:22.297 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:22.329 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:22.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:22.431 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:22.460 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:22.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:22.603 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:22.629 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:22.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:22.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 21041/196368 objects degraded (10.715%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:22.674 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:22 smithi032 ceph-mon[101299]: from='client.45671 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:22:22.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 21041/196368 objects degraded (10.715%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:22.674 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:22 smithi032 ceph-mon[105748]: from='client.45671 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:22:22.678 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:22:22.731 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:22.755 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:22.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 21041/196368 objects degraded (10.715%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:22 smithi045 ceph-mon[95976]: from='client.45671 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:22:22.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:22.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.888 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:22.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:22.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:23.016 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:23.118 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:23.145 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:23.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:23.247 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:23.273 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:23.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:23.375 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:23.410 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:23.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:23.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:23.552 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:23.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:23.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:23.680 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:23.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:23.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:23 smithi032 ceph-mon[101299]: pgmap v1047: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 21041/195630 objects degraded (10.756%); 10969/195630 objects misplaced (5.607%); 12 MiB/s, 185 objects/s recovering 2024-09-21T10:22:23.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:23 smithi032 ceph-mon[105748]: pgmap v1047: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 21041/195630 objects degraded (10.756%); 10969/195630 objects misplaced (5.607%); 12 MiB/s, 185 objects/s recovering 2024-09-21T10:22:23.782 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:23.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.814 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:23.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:23.915 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:23.952 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1046,"stamp":"2024-09-21T10:22:20.516859+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4292614141,"num_objects":65456,"num_object_clones":0,"num_object_copies":196368,"num_objects_missing_on_primary":2902,"num_objects_missing":2902,"num_objects_degraded":21041,"num_objects_misplaced":10969,"num_objects_unfound":0,"num_objects_dirty":65456,"num_whiteouts":0,"num_read":6389,"num_read_kb":13844,"num_write":135381,"num_write_kb":6404930,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16132,"num_bytes_recovered":1044283385,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108602,"ondisk_log_size":108602,"up":297,"acting":288,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17251900,"kb_used_data":12790252,"kb_used_omap":40,"kb_used_meta":4461527,"kb_avail":732709316,"statfs":{"total":767960285184,"available":750294339584,"internally_reserved":0,"allocated":13097218048,"data_stored":13082918156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41824,"internal_metadata":4568603808},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-208732160,"num_objects":-3185,"num_object_clones":0,"num_object_copies":-9555,"num_objects_missing_on_primary":-199,"num_objects_missing":-199,"num_objects_degraded":-627,"num_objects_misplaced":-1579,"num_objects_unfound":0,"num_objects_dirty":-3185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3185,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1579,"num_bytes_recovered":103481344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002065"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":580,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053060+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:22:17.053060+0000","last_peered":"2024-09-21T10:22:17.053060+0000","last_clean":"2024-09-21T10:22:17.053060+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:22:17.053060+0000","last_undegraded":"2024-09-21T10:22:17.053060+0000","last_fullsized":"2024-09-21T10:22:17.053060+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":354,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.065214+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:22:15.065214+0000","last_peered":"2024-09-21T10:22:15.065214+0000","last_clean":"2024-09-21T10:22:15.065214+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:22:15.065214+0000","last_undegraded":"2024-09-21T10:22:15.065214+0000","last_fullsized":"2024-09-21T10:22:15.065214+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":50,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.072958+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:22:15.072958+0000","last_peered":"2024-09-21T10:22:15.072958+0000","last_clean":"2024-09-21T10:22:15.072958+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:22:15.072958+0000","last_undegraded":"2024-09-21T10:22:15.072958+0000","last_fullsized":"2024-09-21T10:22:15.072958+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":20,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.056464+0000","last_change":"2024-09-21T10:22:01.318638+0000","last_active":"2024-09-21T10:22:15.056464+0000","last_peered":"2024-09-21T10:22:15.056464+0000","last_clean":"2024-09-21T10:22:15.056464+0000","last_became_active":"2024-09-21T10:22:01.317087+0000","last_became_peered":"2024-09-21T10:22:01.317087+0000","last_unstale":"2024-09-21T10:22:15.056464+0000","last_undegraded":"2024-09-21T10:22:15.056464+0000","last_fullsized":"2024-09-21T10:22:15.056464+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:25.334726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":21,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.056849+0000","last_change":"2024-09-21T10:22:01.318978+0000","last_active":"2024-09-21T10:22:15.056849+0000","last_peered":"2024-09-21T10:22:15.056849+0000","last_clean":"2024-09-21T10:22:15.056849+0000","last_became_active":"2024-09-21T10:22:01.318255+0000","last_became_peered":"2024-09-21T10:22:01.318255+0000","last_unstale":"2024-09-21T10:22:15.056849+0000","last_undegraded":"2024-09-21T10:22:15.056849+0000","last_fullsized":"2024-09-21T10:22:15.056849+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:55.084088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":325,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.064914+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:22:15.064914+0000","last_peered":"2024-09-21T10:22:15.064914+0000","last_clean":"2024-09-21T10:22:15.064914+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:22:15.064914+0000","last_undegraded":"2024-09-21T10:22:15.064914+0000","last_fullsized":"2024-09-21T10:22:15.064914+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":601,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054082+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:22:17.054082+0000","last_peered":"2024-09-21T10:22:17.054082+0000","last_clean":"2024-09-21T10:22:17.054082+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:22:17.054082+0000","last_undegraded":"2024-09-21T10:22:17.054082+0000","last_fullsized":"2024-09-21T10:22:17.054082+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":406,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.049487+0000","last_change":"2024-09-21T10:22:01.331612+0000","last_active":"2024-09-21T10:22:15.049487+0000","last_peered":"2024-09-21T10:22:15.049487+0000","last_clean":"2024-09-21T10:22:15.049487+0000","last_became_active":"2024-09-21T10:22:01.325282+0000","last_became_peered":"2024-09-21T10:22:01.325282+0000","last_unstale":"2024-09-21T10:22:15.049487+0000","last_undegraded":"2024-09-21T10:22:15.049487+0000","last_fullsized":"2024-09-21T10:22:15.049487+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:23.383106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"352'8445","reported_seq":9504,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:19.901404+0000","last_change":"2024-09-21T10:21:39.510930+0000","last_active":"2024-09-21T10:22:19.901404+0000","last_peered":"2024-09-21T10:22:19.901404+0000","last_clean":"2024-09-21T10:20:12.056605+0000","last_became_active":"2024-09-21T10:20:14.466147+0000","last_became_peered":"2024-09-21T10:20:14.466147+0000","last_unstale":"2024-09-21T10:22:19.901404+0000","last_undegraded":"2024-09-21T10:20:14.094800+0000","last_fullsized":"2024-09-21T10:20:14.071992+0000","mapping_epoch":325,"log_start":"314'1100","ondisk_log_start":"314'1100","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7345,"log_dups_size":0,"ondisk_log_size":7345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:34.237367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267059200,"num_objects":4075,"num_object_clones":0,"num_object_copies":12225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3167,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8445,"num_write_kb":400640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":489,"num_bytes_recovered":32047104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":908},{"shards":"2,4,6","objects":1470}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":41,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.065390+0000","last_change":"2024-09-21T10:22:01.321467+0000","last_active":"2024-09-21T10:22:15.065390+0000","last_peered":"2024-09-21T10:22:15.065390+0000","last_clean":"2024-09-21T10:22:15.065390+0000","last_became_active":"2024-09-21T10:22:01.318270+0000","last_became_peered":"2024-09-21T10:22:01.318270+0000","last_unstale":"2024-09-21T10:22:15.065390+0000","last_undegraded":"2024-09-21T10:22:15.065390+0000","last_fullsized":"2024-09-21T10:22:15.065390+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:44.059553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":59,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.051207+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:22:17.051207+0000","last_peered":"2024-09-21T10:22:17.051207+0000","last_clean":"2024-09-21T10:22:17.051207+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:22:17.051207+0000","last_undegraded":"2024-09-21T10:22:17.051207+0000","last_fullsized":"2024-09-21T10:22:17.051207+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"352'8238","reported_seq":12105,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:19.883951+0000","last_change":"2024-09-21T10:20:04.684629+0000","last_active":"2024-09-21T10:22:19.883951+0000","last_peered":"2024-09-21T10:22:19.883951+0000","last_clean":"2024-09-21T10:22:19.883951+0000","last_became_active":"2024-09-21T10:20:04.680299+0000","last_became_peered":"2024-09-21T10:20:04.680299+0000","last_unstale":"2024-09-21T10:22:19.883951+0000","last_undegraded":"2024-09-21T10:22:19.883951+0000","last_fullsized":"2024-09-21T10:22:19.883951+0000","mapping_epoch":320,"log_start":"314'1700","ondisk_log_start":"314'1700","created":305,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6538,"log_dups_size":0,"ondisk_log_size":6538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:41.743081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262930432,"num_objects":4012,"num_object_clones":0,"num_object_copies":12036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8238,"num_write_kb":392000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1891,"num_bytes_recovered":123928576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":28,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.065844+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:22:15.065844+0000","last_peered":"2024-09-21T10:22:15.065844+0000","last_clean":"2024-09-21T10:22:15.065844+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:22:15.065844+0000","last_undegraded":"2024-09-21T10:22:15.065844+0000","last_fullsized":"2024-09-21T10:22:15.065844+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":48,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.051456+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:22:15.051456+0000","last_peered":"2024-09-21T10:22:15.051456+0000","last_clean":"2024-09-21T10:22:15.051456+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:22:15.051456+0000","last_undegraded":"2024-09-21T10:22:15.051456+0000","last_fullsized":"2024-09-21T10:22:15.051456+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"351'8195","reported_seq":8299,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:16.593408+0000","last_change":"2024-09-21T10:19:22.649712+0000","last_active":"2024-09-21T10:22:16.593408+0000","last_peered":"2024-09-21T10:22:16.593408+0000","last_clean":"2024-09-21T10:22:16.593408+0000","last_became_active":"2024-09-21T10:19:22.649303+0000","last_became_peered":"2024-09-21T10:19:22.649303+0000","last_unstale":"2024-09-21T10:22:16.593408+0000","last_undegraded":"2024-09-21T10:22:16.593408+0000","last_fullsized":"2024-09-21T10:22:16.593408+0000","mapping_epoch":305,"log_start":"314'961","ondisk_log_start":"314'961","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7234,"log_dups_size":0,"ondisk_log_size":7234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:24.442851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269287424,"num_objects":4109,"num_object_clones":0,"num_object_copies":12327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8195,"num_write_kb":393728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":350,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.065917+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:22:15.065917+0000","last_peered":"2024-09-21T10:22:15.065917+0000","last_clean":"2024-09-21T10:22:15.065917+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:22:15.065917+0000","last_undegraded":"2024-09-21T10:22:15.065917+0000","last_fullsized":"2024-09-21T10:22:15.065917+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":369,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.049475+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:22:15.049475+0000","last_peered":"2024-09-21T10:22:15.049475+0000","last_clean":"2024-09-21T10:22:15.049475+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:22:15.049475+0000","last_undegraded":"2024-09-21T10:22:15.049475+0000","last_fullsized":"2024-09-21T10:22:15.049475+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"352'8660","reported_seq":8766,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:20.326654+0000","last_change":"2024-09-21T10:19:22.640645+0000","last_active":"2024-09-21T10:22:20.326654+0000","last_peered":"2024-09-21T10:22:20.326654+0000","last_clean":"2024-09-21T10:22:20.326654+0000","last_became_active":"2024-09-21T10:19:22.640343+0000","last_became_peered":"2024-09-21T10:19:22.640343+0000","last_unstale":"2024-09-21T10:22:20.326654+0000","last_undegraded":"2024-09-21T10:22:20.326654+0000","last_fullsized":"2024-09-21T10:22:20.326654+0000","mapping_epoch":305,"log_start":"321'3300","ondisk_log_start":"321'3300","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:48.734122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274071552,"num_objects":4182,"num_object_clones":0,"num_object_copies":12546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8660,"num_write_kb":410944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":283,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.048717+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:22:15.048717+0000","last_peered":"2024-09-21T10:22:15.048717+0000","last_clean":"2024-09-21T10:22:15.048717+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:22:15.048717+0000","last_undegraded":"2024-09-21T10:22:15.048717+0000","last_fullsized":"2024-09-21T10:22:15.048717+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":190,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.056692+0000","last_change":"2024-09-21T10:19:50.585724+0000","last_active":"2024-09-21T10:22:15.056692+0000","last_peered":"2024-09-21T10:22:15.056692+0000","last_clean":"2024-09-21T10:22:15.056692+0000","last_became_active":"2024-09-21T10:19:50.585315+0000","last_became_peered":"2024-09-21T10:19:50.585315+0000","last_unstale":"2024-09-21T10:22:15.056692+0000","last_undegraded":"2024-09-21T10:22:15.056692+0000","last_fullsized":"2024-09-21T10:22:15.056692+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:26.618970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"352'8380","reported_seq":13449,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:18.734312+0000","last_change":"2024-09-21T10:22:01.328242+0000","last_active":"2024-09-21T10:22:18.734312+0000","last_peered":"2024-09-21T10:22:18.734312+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492359+0000","last_became_peered":"2024-09-21T10:20:14.492359+0000","last_unstale":"2024-09-21T10:22:18.734312+0000","last_undegraded":"2024-09-21T10:20:14.218670+0000","last_fullsized":"2024-09-21T10:20:14.071510+0000","mapping_epoch":325,"log_start":"314'1500","ondisk_log_start":"314'1500","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6880,"log_dups_size":0,"ondisk_log_size":6880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271581184,"num_objects":4144,"num_object_clones":0,"num_object_copies":12432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8380,"num_write_kb":400768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2481,"num_bytes_recovered":148307968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":2940}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":606,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.062439+0000","last_change":"2024-09-21T10:22:04.029688+0000","last_active":"2024-09-21T10:22:15.062439+0000","last_peered":"2024-09-21T10:22:15.062439+0000","last_clean":"2024-09-21T10:22:15.062439+0000","last_became_active":"2024-09-21T10:22:04.028853+0000","last_became_peered":"2024-09-21T10:22:04.028853+0000","last_unstale":"2024-09-21T10:22:15.062439+0000","last_undegraded":"2024-09-21T10:22:15.062439+0000","last_fullsized":"2024-09-21T10:22:15.062439+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:20.457596+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:19:20.457596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":631,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054741+0000","last_change":"2024-09-21T10:22:01.335303+0000","last_active":"2024-09-21T10:22:17.054741+0000","last_peered":"2024-09-21T10:22:17.054741+0000","last_clean":"2024-09-21T10:22:17.054741+0000","last_became_active":"2024-09-21T10:22:01.325298+0000","last_became_peered":"2024-09-21T10:22:01.325298+0000","last_unstale":"2024-09-21T10:22:17.054741+0000","last_undegraded":"2024-09-21T10:22:17.054741+0000","last_fullsized":"2024-09-21T10:22:17.054741+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:01.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":48,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.052393+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:22:15.052393+0000","last_peered":"2024-09-21T10:22:15.052393+0000","last_clean":"2024-09-21T10:22:15.052393+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:22:15.052393+0000","last_undegraded":"2024-09-21T10:22:15.052393+0000","last_fullsized":"2024-09-21T10:22:15.052393+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"352'8285","reported_seq":8391,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:20.337689+0000","last_change":"2024-09-21T10:19:22.641281+0000","last_active":"2024-09-21T10:22:20.337689+0000","last_peered":"2024-09-21T10:22:20.337689+0000","last_clean":"2024-09-21T10:22:20.337689+0000","last_became_active":"2024-09-21T10:19:22.640988+0000","last_became_peered":"2024-09-21T10:19:22.640988+0000","last_unstale":"2024-09-21T10:22:20.337689+0000","last_undegraded":"2024-09-21T10:22:20.337689+0000","last_fullsized":"2024-09-21T10:22:20.337689+0000","mapping_epoch":305,"log_start":"319'3000","ondisk_log_start":"319'3000","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5285,"log_dups_size":0,"ondisk_log_size":5285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:52.652565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256704512,"num_objects":3917,"num_object_clones":0,"num_object_copies":11751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8285,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":420,"reported_epoch":351,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:15.062485+0000","last_change":"2024-09-21T10:22:02.013601+0000","last_active":"2024-09-21T10:22:15.062485+0000","last_peered":"2024-09-21T10:22:15.062485+0000","last_clean":"2024-09-21T10:19:15.364615+0000","last_became_active":"2024-09-21T10:22:02.013140+0000","last_became_peered":"2024-09-21T10:22:02.013140+0000","last_unstale":"2024-09-21T10:22:15.062485+0000","last_undegraded":"2024-09-21T10:22:02.004849+0000","last_fullsized":"2024-09-21T10:22:02.002552+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_clean_scrub_stamp":"2024-09-21T10:19:15.365774+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1},{"shards":"6,7","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":359,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.049610+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:22:15.049610+0000","last_peered":"2024-09-21T10:22:15.049610+0000","last_clean":"2024-09-21T10:22:15.049610+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:22:15.049610+0000","last_undegraded":"2024-09-21T10:22:15.049610+0000","last_fullsized":"2024-09-21T10:22:15.049610+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":48,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.052004+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:22:15.052004+0000","last_peered":"2024-09-21T10:22:15.052004+0000","last_clean":"2024-09-21T10:22:15.052004+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:22:15.052004+0000","last_undegraded":"2024-09-21T10:22:15.052004+0000","last_fullsized":"2024-09-21T10:22:15.052004+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"352'8459","reported_seq":11774,"reported_epoch":352,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:19.971422+0000","last_change":"2024-09-21T10:22:01.331955+0000","last_active":"2024-09-21T10:22:19.971422+0000","last_peered":"2024-09-21T10:22:19.971422+0000","last_clean":"2024-09-21T10:20:12.057304+0000","last_became_active":"2024-09-21T10:20:14.547332+0000","last_became_peered":"2024-09-21T10:20:14.547332+0000","last_unstale":"2024-09-21T10:22:19.971422+0000","last_undegraded":"2024-09-21T10:20:14.094983+0000","last_fullsized":"2024-09-21T10:20:14.073184+0000","mapping_epoch":325,"log_start":"314'1100","ondisk_log_start":"314'1100","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7359,"log_dups_size":0,"ondisk_log_size":7359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:10.148038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272302080,"num_objects":4155,"num_object_clones":0,"num_object_copies":12465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1762,"num_objects_misplaced":237,"num_objects_unfound":0,"num_objects_dirty":4155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8459,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1619,"num_bytes_recovered":106102784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":237},{"shards":"0,4","objects":2156}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":48,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.051362+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:22:15.051362+0000","last_peered":"2024-09-21T10:22:15.051362+0000","last_clean":"2024-09-21T10:22:15.051362+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:22:15.051362+0000","last_undegraded":"2024-09-21T10:22:15.051362+0000","last_fullsized":"2024-09-21T10:22:15.051362+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":48,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.052056+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:22:15.052056+0000","last_peered":"2024-09-21T10:22:15.052056+0000","last_clean":"2024-09-21T10:22:15.052056+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:22:15.052056+0000","last_undegraded":"2024-09-21T10:22:15.052056+0000","last_fullsized":"2024-09-21T10:22:15.052056+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"352'8389","reported_seq":8508,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:20.151719+0000","last_change":"2024-09-21T10:21:39.558048+0000","last_active":"2024-09-21T10:22:20.151719+0000","last_peered":"2024-09-21T10:22:20.151719+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492538+0000","last_became_peered":"2024-09-21T10:20:14.492538+0000","last_unstale":"2024-09-21T10:22:20.151719+0000","last_undegraded":"2024-09-21T10:20:14.216051+0000","last_fullsized":"2024-09-21T10:20:14.075057+0000","mapping_epoch":325,"log_start":"321'3100","ondisk_log_start":"321'3100","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5289,"log_dups_size":0,"ondisk_log_size":5289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270860288,"num_objects":4133,"num_object_clones":0,"num_object_copies":12399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8389,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":557}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":606,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.064966+0000","last_change":"2024-09-21T10:22:04.031320+0000","last_active":"2024-09-21T10:22:15.064966+0000","last_peered":"2024-09-21T10:22:15.064966+0000","last_clean":"2024-09-21T10:22:15.064966+0000","last_became_active":"2024-09-21T10:22:04.030774+0000","last_became_peered":"2024-09-21T10:22:04.030774+0000","last_unstale":"2024-09-21T10:22:15.064966+0000","last_undegraded":"2024-09-21T10:22:15.064966+0000","last_fullsized":"2024-09-21T10:22:15.064966+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:22.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:22.489745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":609,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052533+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:22:17.052533+0000","last_peered":"2024-09-21T10:22:17.052533+0000","last_clean":"2024-09-21T10:22:17.052533+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:22:17.052533+0000","last_undegraded":"2024-09-21T10:22:17.052533+0000","last_fullsized":"2024-09-21T10:22:17.052533+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":302,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.055265+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:22:17.055265+0000","last_peered":"2024-09-21T10:22:17.055265+0000","last_clean":"2024-09-21T10:22:17.055265+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:22:17.055265+0000","last_undegraded":"2024-09-21T10:22:17.055265+0000","last_fullsized":"2024-09-21T10:22:17.055265+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"352'8416","reported_seq":9295,"reported_epoch":352,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:22:19.886782+0000","last_change":"2024-09-21T10:22:02.106012+0000","last_active":"2024-09-21T10:22:19.886782+0000","last_peered":"2024-09-21T10:22:19.886782+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:22:02.086281+0000","last_became_peered":"2024-09-21T10:22:02.086281+0000","last_unstale":"2024-09-21T10:22:19.886782+0000","last_undegraded":"2024-09-21T10:22:02.081791+0000","last_fullsized":"2024-09-21T10:22:19.886782+0000","mapping_epoch":347,"log_start":"314'1800","ondisk_log_start":"314'1800","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6616,"log_dups_size":0,"ondisk_log_size":6616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:37.553227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263847936,"num_objects":4026,"num_object_clones":0,"num_object_copies":12078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3269,"num_objects_misplaced":4026,"num_objects_unfound":0,"num_objects_dirty":4026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8416,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":402,"num_bytes_recovered":26345472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,5,6","objects":2550},{"shards":"3,6","objects":1476}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":606,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.062447+0000","last_change":"2024-09-21T10:22:04.029284+0000","last_active":"2024-09-21T10:22:15.062447+0000","last_peered":"2024-09-21T10:22:15.062447+0000","last_clean":"2024-09-21T10:22:15.062447+0000","last_became_active":"2024-09-21T10:22:04.028938+0000","last_became_peered":"2024-09-21T10:22:04.028938+0000","last_unstale":"2024-09-21T10:22:15.062447+0000","last_undegraded":"2024-09-21T10:22:15.062447+0000","last_fullsized":"2024-09-21T10:22:15.062447+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:12.388304+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:12.388304+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":636,"reported_epoch":351,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:15.048389+0000","last_change":"2024-09-21T10:21:36.010162+0000","last_active":"2024-09-21T10:22:15.048389+0000","last_peered":"2024-09-21T10:22:15.048389+0000","last_clean":"2024-09-21T10:21:17.725138+0000","last_became_active":"2024-09-21T10:21:36.009706+0000","last_became_peered":"2024-09-21T10:21:36.009706+0000","last_unstale":"2024-09-21T10:22:15.048389+0000","last_undegraded":"2024-09-21T10:21:35.852171+0000","last_fullsized":"2024-09-21T10:21:35.851526+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:04.206298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":540,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053470+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:22:17.053470+0000","last_peered":"2024-09-21T10:22:17.053470+0000","last_clean":"2024-09-21T10:22:17.053470+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:22:17.053470+0000","last_undegraded":"2024-09-21T10:22:17.053470+0000","last_fullsized":"2024-09-21T10:22:17.053470+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":326,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.050082+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:22:15.050082+0000","last_peered":"2024-09-21T10:22:15.050082+0000","last_clean":"2024-09-21T10:22:15.050082+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:22:15.050082+0000","last_undegraded":"2024-09-21T10:22:15.050082+0000","last_fullsized":"2024-09-21T10:22:15.050082+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"352'8241","reported_seq":8347,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:20.208255+0000","last_change":"2024-09-21T10:20:14.480108+0000","last_active":"2024-09-21T10:22:20.208255+0000","last_peered":"2024-09-21T10:22:20.208255+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.471864+0000","last_became_peered":"2024-09-21T10:20:14.471864+0000","last_unstale":"2024-09-21T10:22:20.208255+0000","last_undegraded":"2024-09-21T10:20:14.214064+0000","last_fullsized":"2024-09-21T10:20:14.074889+0000","mapping_epoch":325,"log_start":"319'2900","ondisk_log_start":"319'2900","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5341,"log_dups_size":0,"ondisk_log_size":5341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259719168,"num_objects":3963,"num_object_clones":0,"num_object_copies":11889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8241,"num_write_kb":390528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":418}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":606,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.064877+0000","last_change":"2024-09-21T10:22:04.031573+0000","last_active":"2024-09-21T10:22:15.064877+0000","last_peered":"2024-09-21T10:22:15.064877+0000","last_clean":"2024-09-21T10:22:15.064877+0000","last_became_active":"2024-09-21T10:22:04.031125+0000","last_became_peered":"2024-09-21T10:22:04.031125+0000","last_unstale":"2024-09-21T10:22:15.064877+0000","last_undegraded":"2024-09-21T10:22:15.064877+0000","last_fullsized":"2024-09-21T10:22:15.064877+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:19:19.450903+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":991,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.062592+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:22:15.062592+0000","last_peered":"2024-09-21T10:22:15.062592+0000","last_clean":"2024-09-21T10:22:15.062592+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:22:15.062592+0000","last_undegraded":"2024-09-21T10:22:15.062592+0000","last_fullsized":"2024-09-21T10:22:15.062592+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":188,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.056542+0000","last_change":"2024-09-21T10:19:51.555124+0000","last_active":"2024-09-21T10:22:15.056542+0000","last_peered":"2024-09-21T10:22:15.056542+0000","last_clean":"2024-09-21T10:22:15.056542+0000","last_became_active":"2024-09-21T10:19:51.554766+0000","last_became_peered":"2024-09-21T10:19:51.554766+0000","last_unstale":"2024-09-21T10:22:15.056542+0000","last_undegraded":"2024-09-21T10:22:15.056542+0000","last_fullsized":"2024-09-21T10:22:15.056542+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:52.832950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":20,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.056759+0000","last_change":"2024-09-21T10:22:01.316140+0000","last_active":"2024-09-21T10:22:15.056759+0000","last_peered":"2024-09-21T10:22:15.056759+0000","last_clean":"2024-09-21T10:22:15.056759+0000","last_became_active":"2024-09-21T10:22:01.315408+0000","last_became_peered":"2024-09-21T10:22:01.315408+0000","last_unstale":"2024-09-21T10:22:15.056759+0000","last_undegraded":"2024-09-21T10:22:15.056759+0000","last_fullsized":"2024-09-21T10:22:15.056759+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:07.322279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"352'8431","reported_seq":12398,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:18.566085+0000","last_change":"2024-09-21T10:20:47.589372+0000","last_active":"2024-09-21T10:22:18.566085+0000","last_peered":"2024-09-21T10:22:18.566085+0000","last_clean":"2024-09-21T10:22:18.566085+0000","last_became_active":"2024-09-21T10:20:47.580051+0000","last_became_peered":"2024-09-21T10:20:47.580051+0000","last_unstale":"2024-09-21T10:22:18.566085+0000","last_undegraded":"2024-09-21T10:22:18.566085+0000","last_fullsized":"2024-09-21T10:22:18.566085+0000","mapping_epoch":332,"log_start":"314'1600","ondisk_log_start":"314'1600","created":305,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6831,"log_dups_size":0,"ondisk_log_size":6831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:46.363318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273088512,"num_objects":4167,"num_object_clones":0,"num_object_copies":12501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8431,"num_write_kb":403136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":606,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.064884+0000","last_change":"2024-09-21T10:22:04.030341+0000","last_active":"2024-09-21T10:22:15.064884+0000","last_peered":"2024-09-21T10:22:15.064884+0000","last_clean":"2024-09-21T10:22:15.064884+0000","last_became_active":"2024-09-21T10:22:04.030000+0000","last_became_peered":"2024-09-21T10:22:04.030000+0000","last_unstale":"2024-09-21T10:22:15.064884+0000","last_undegraded":"2024-09-21T10:22:15.064884+0000","last_fullsized":"2024-09-21T10:22:15.064884+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:10.332770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:10.332770+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1130,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.059659+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:22:17.059659+0000","last_peered":"2024-09-21T10:22:17.059659+0000","last_clean":"2024-09-21T10:22:17.059659+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:22:17.059659+0000","last_undegraded":"2024-09-21T10:22:17.059659+0000","last_fullsized":"2024-09-21T10:22:17.059659+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":288,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.051931+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:22:17.051931+0000","last_peered":"2024-09-21T10:22:17.051931+0000","last_clean":"2024-09-21T10:22:17.051931+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:22:17.051931+0000","last_undegraded":"2024-09-21T10:22:17.051931+0000","last_fullsized":"2024-09-21T10:22:17.051931+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":318,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.055333+0000","last_change":"2024-09-21T10:22:01.317933+0000","last_active":"2024-09-21T10:22:17.055333+0000","last_peered":"2024-09-21T10:22:17.055333+0000","last_clean":"2024-09-21T10:22:17.055333+0000","last_became_active":"2024-09-21T10:22:01.317536+0000","last_became_peered":"2024-09-21T10:22:01.317536+0000","last_unstale":"2024-09-21T10:22:17.055333+0000","last_undegraded":"2024-09-21T10:22:17.055333+0000","last_fullsized":"2024-09-21T10:22:17.055333+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:34.312724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"352'8431","reported_seq":15623,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:18.771584+0000","last_change":"2024-09-21T10:21:34.868559+0000","last_active":"2024-09-21T10:22:18.771584+0000","last_peered":"2024-09-21T10:22:18.771584+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.463996+0000","last_became_peered":"2024-09-21T10:20:14.463996+0000","last_unstale":"2024-09-21T10:22:18.771584+0000","last_undegraded":"2024-09-21T10:20:14.217266+0000","last_fullsized":"2024-09-21T10:20:14.071353+0000","mapping_epoch":325,"log_start":"314'1600","ondisk_log_start":"314'1600","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6831,"log_dups_size":0,"ondisk_log_size":6831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277807104,"num_objects":4239,"num_object_clones":0,"num_object_copies":12717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":98,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8431,"num_write_kb":405440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":232128512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":4141}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":606,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.064758+0000","last_change":"2024-09-21T10:22:04.029892+0000","last_active":"2024-09-21T10:22:15.064758+0000","last_peered":"2024-09-21T10:22:15.064758+0000","last_clean":"2024-09-21T10:22:15.064758+0000","last_became_active":"2024-09-21T10:22:04.029353+0000","last_became_peered":"2024-09-21T10:22:04.029353+0000","last_unstale":"2024-09-21T10:22:15.064758+0000","last_undegraded":"2024-09-21T10:22:15.064758+0000","last_fullsized":"2024-09-21T10:22:15.064758+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:19:13.371825+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1094,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.048808+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:22:15.048808+0000","last_peered":"2024-09-21T10:22:15.048808+0000","last_clean":"2024-09-21T10:22:15.048808+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:22:15.048808+0000","last_undegraded":"2024-09-21T10:22:15.048808+0000","last_fullsized":"2024-09-21T10:22:15.048808+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":48,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.051820+0000","last_change":"2024-09-21T10:19:50.581529+0000","last_active":"2024-09-21T10:22:15.051820+0000","last_peered":"2024-09-21T10:22:15.051820+0000","last_clean":"2024-09-21T10:22:15.051820+0000","last_became_active":"2024-09-21T10:19:50.581119+0000","last_became_peered":"2024-09-21T10:19:50.581119+0000","last_unstale":"2024-09-21T10:22:15.051820+0000","last_undegraded":"2024-09-21T10:22:15.051820+0000","last_fullsized":"2024-09-21T10:22:15.051820+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:13.334483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":288,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054626+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:22:17.054626+0000","last_peered":"2024-09-21T10:22:17.054626+0000","last_clean":"2024-09-21T10:22:17.054626+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:22:17.054626+0000","last_undegraded":"2024-09-21T10:22:17.054626+0000","last_fullsized":"2024-09-21T10:22:17.054626+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"351'8509","reported_seq":12602,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:16.515988+0000","last_change":"2024-09-21T10:20:42.601607+0000","last_active":"2024-09-21T10:22:16.515988+0000","last_peered":"2024-09-21T10:22:16.515988+0000","last_clean":"2024-09-21T10:22:16.515988+0000","last_became_active":"2024-09-21T10:20:42.593406+0000","last_became_peered":"2024-09-21T10:20:42.593406+0000","last_unstale":"2024-09-21T10:22:16.515988+0000","last_undegraded":"2024-09-21T10:22:16.515988+0000","last_fullsized":"2024-09-21T10:22:16.515988+0000","mapping_epoch":330,"log_start":"314'1268","ondisk_log_start":"314'1268","created":305,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7241,"log_dups_size":0,"ondisk_log_size":7241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:09.770489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276103168,"num_objects":4213,"num_object_clones":0,"num_object_copies":12639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8509,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1985,"num_bytes_recovered":130088960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":606,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.065520+0000","last_change":"2024-09-21T10:22:04.029371+0000","last_active":"2024-09-21T10:22:15.065520+0000","last_peered":"2024-09-21T10:22:15.065520+0000","last_clean":"2024-09-21T10:22:15.065520+0000","last_became_active":"2024-09-21T10:22:04.029014+0000","last_became_peered":"2024-09-21T10:22:04.029014+0000","last_unstale":"2024-09-21T10:22:15.065520+0000","last_undegraded":"2024-09-21T10:22:15.065520+0000","last_fullsized":"2024-09-21T10:22:15.065520+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":299,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067093+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:22:17.067093+0000","last_peered":"2024-09-21T10:22:17.067093+0000","last_clean":"2024-09-21T10:22:17.067093+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:22:17.067093+0000","last_undegraded":"2024-09-21T10:22:17.067093+0000","last_fullsized":"2024-09-21T10:22:17.067093+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":897,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054819+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:22:17.054819+0000","last_peered":"2024-09-21T10:22:17.054819+0000","last_clean":"2024-09-21T10:22:17.054819+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:22:17.054819+0000","last_undegraded":"2024-09-21T10:22:17.054819+0000","last_fullsized":"2024-09-21T10:22:17.054819+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":324,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053780+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:22:17.053780+0000","last_peered":"2024-09-21T10:22:17.053780+0000","last_clean":"2024-09-21T10:22:17.053780+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:22:17.053780+0000","last_undegraded":"2024-09-21T10:22:17.053780+0000","last_fullsized":"2024-09-21T10:22:17.053780+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"352'8334","reported_seq":8440,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:18.788050+0000","last_change":"2024-09-21T10:19:22.641043+0000","last_active":"2024-09-21T10:22:18.788050+0000","last_peered":"2024-09-21T10:22:18.788050+0000","last_clean":"2024-09-21T10:22:18.788050+0000","last_became_active":"2024-09-21T10:19:22.640742+0000","last_became_peered":"2024-09-21T10:19:22.640742+0000","last_unstale":"2024-09-21T10:22:18.788050+0000","last_undegraded":"2024-09-21T10:22:18.788050+0000","last_fullsized":"2024-09-21T10:22:18.788050+0000","mapping_epoch":305,"log_start":"314'1500","ondisk_log_start":"314'1500","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6834,"log_dups_size":0,"ondisk_log_size":6834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:36.901208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262930432,"num_objects":4012,"num_object_clones":0,"num_object_copies":12036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8334,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":606,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.065919+0000","last_change":"2024-09-21T10:22:04.031872+0000","last_active":"2024-09-21T10:22:15.065919+0000","last_peered":"2024-09-21T10:22:15.065919+0000","last_clean":"2024-09-21T10:22:15.065919+0000","last_became_active":"2024-09-21T10:22:04.031343+0000","last_became_peered":"2024-09-21T10:22:04.031343+0000","last_unstale":"2024-09-21T10:22:15.065919+0000","last_undegraded":"2024-09-21T10:22:15.065919+0000","last_fullsized":"2024-09-21T10:22:15.065919+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:07.332404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:07.332404+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":306,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054741+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:22:17.054741+0000","last_peered":"2024-09-21T10:22:17.054741+0000","last_clean":"2024-09-21T10:22:17.054741+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:22:17.054741+0000","last_undegraded":"2024-09-21T10:22:17.054741+0000","last_fullsized":"2024-09-21T10:22:17.054741+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":917,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.065999+0000","last_change":"2024-09-21T10:22:01.336961+0000","last_active":"2024-09-21T10:22:15.065999+0000","last_peered":"2024-09-21T10:22:15.065999+0000","last_clean":"2024-09-21T10:22:15.065999+0000","last_became_active":"2024-09-21T10:22:01.320079+0000","last_became_peered":"2024-09-21T10:22:01.320079+0000","last_unstale":"2024-09-21T10:22:15.065999+0000","last_undegraded":"2024-09-21T10:22:15.065999+0000","last_fullsized":"2024-09-21T10:22:15.065999+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:54.037388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":187,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.056615+0000","last_change":"2024-09-21T10:20:13.088557+0000","last_active":"2024-09-21T10:22:15.056615+0000","last_peered":"2024-09-21T10:22:15.056615+0000","last_clean":"2024-09-21T10:22:15.056615+0000","last_became_active":"2024-09-21T10:20:13.088123+0000","last_became_peered":"2024-09-21T10:20:13.088123+0000","last_unstale":"2024-09-21T10:22:15.056615+0000","last_undegraded":"2024-09-21T10:22:15.056615+0000","last_fullsized":"2024-09-21T10:22:15.056615+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:49.475499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"352'8459","reported_seq":10123,"reported_epoch":352,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:22:18.803433+0000","last_change":"2024-09-21T10:22:01.405956+0000","last_active":"2024-09-21T10:22:18.803433+0000","last_peered":"2024-09-21T10:22:18.803433+0000","last_clean":"2024-09-21T10:20:12.067444+0000","last_became_active":"2024-09-21T10:22:01.391808+0000","last_became_peered":"2024-09-21T10:22:01.391808+0000","last_unstale":"2024-09-21T10:22:18.803433+0000","last_undegraded":"2024-09-21T10:22:01.123056+0000","last_fullsized":"2024-09-21T10:22:18.803433+0000","mapping_epoch":346,"log_start":"314'1600","ondisk_log_start":"314'1600","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6859,"log_dups_size":0,"ondisk_log_size":6859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268500992,"num_objects":4097,"num_object_clones":0,"num_object_copies":12291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2945,"num_objects_misplaced":5249,"num_objects_unfound":0,"num_objects_dirty":4097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8459,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":768,"num_bytes_recovered":50200576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[3,5,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,5","objects":2565},{"shards":"3,4,6","objects":1532}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":606,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.065743+0000","last_change":"2024-09-21T10:22:04.032207+0000","last_active":"2024-09-21T10:22:15.065743+0000","last_peered":"2024-09-21T10:22:15.065743+0000","last_clean":"2024-09-21T10:22:15.065743+0000","last_became_active":"2024-09-21T10:22:04.031881+0000","last_became_peered":"2024-09-21T10:22:04.031881+0000","last_unstale":"2024-09-21T10:22:15.065743+0000","last_undegraded":"2024-09-21T10:22:15.065743+0000","last_fullsized":"2024-09-21T10:22:15.065743+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:08.332410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:08.332410+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":614,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054434+0000","last_change":"2024-09-21T10:21:34.876196+0000","last_active":"2024-09-21T10:22:17.054434+0000","last_peered":"2024-09-21T10:22:17.054434+0000","last_clean":"2024-09-21T10:22:17.054434+0000","last_became_active":"2024-09-21T10:21:34.868031+0000","last_became_peered":"2024-09-21T10:21:34.868031+0000","last_unstale":"2024-09-21T10:22:17.054434+0000","last_undegraded":"2024-09-21T10:22:17.054434+0000","last_fullsized":"2024-09-21T10:22:17.054434+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:58.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":960,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054412+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:22:17.054412+0000","last_peered":"2024-09-21T10:22:17.054412+0000","last_clean":"2024-09-21T10:22:17.054412+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:22:17.054412+0000","last_undegraded":"2024-09-21T10:22:17.054412+0000","last_fullsized":"2024-09-21T10:22:17.054412+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":610,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052378+0000","last_change":"2024-09-21T10:22:01.329400+0000","last_active":"2024-09-21T10:22:17.052378+0000","last_peered":"2024-09-21T10:22:17.052378+0000","last_clean":"2024-09-21T10:22:17.052378+0000","last_became_active":"2024-09-21T10:22:01.328887+0000","last_became_peered":"2024-09-21T10:22:01.328887+0000","last_unstale":"2024-09-21T10:22:17.052378+0000","last_undegraded":"2024-09-21T10:22:17.052378+0000","last_fullsized":"2024-09-21T10:22:17.052378+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:09.626637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"352'8386","reported_seq":9995,"reported_epoch":352,"state":"active+recovering+degraded+remapped","last_fresh":"2024-09-21T10:22:19.619391+0000","last_change":"2024-09-21T10:21:39.570426+0000","last_active":"2024-09-21T10:22:19.619391+0000","last_peered":"2024-09-21T10:22:19.619391+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:21:39.424206+0000","last_became_peered":"2024-09-21T10:21:39.424206+0000","last_unstale":"2024-09-21T10:22:19.619391+0000","last_undegraded":"2024-09-21T10:21:39.368782+0000","last_fullsized":"2024-09-21T10:22:19.619391+0000","mapping_epoch":339,"log_start":"313'300","ondisk_log_start":"313'300","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":8086,"log_dups_size":0,"ondisk_log_size":8086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:54.426685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261029912,"num_objects":3984,"num_object_clones":0,"num_object_copies":11952,"num_objects_missing_on_primary":2897,"num_objects_missing":2897,"num_objects_degraded":1450,"num_objects_misplaced":1457,"num_objects_unfound":0,"num_objects_dirty":3984,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8386,"num_write_kb":395777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":754,"num_bytes_recovered":49152000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,5,6","objects":2527},{"shards":"2,6","objects":1450}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":581,"reported_epoch":351,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:15.065300+0000","last_change":"2024-09-21T10:22:02.009815+0000","last_active":"2024-09-21T10:22:15.065300+0000","last_peered":"2024-09-21T10:22:15.065300+0000","last_clean":"2024-09-21T10:12:59.519603+0000","last_became_active":"2024-09-21T10:22:02.009463+0000","last_became_peered":"2024-09-21T10:22:02.009463+0000","last_unstale":"2024-09-21T10:22:15.065300+0000","last_undegraded":"2024-09-21T10:22:02.002510+0000","last_fullsized":"2024-09-21T10:22:02.001835+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:02.435487+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:02.435487+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1},{"shards":"6,7","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":17,"reported_epoch":351,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:15.056776+0000","last_change":"2024-09-21T10:22:01.318966+0000","last_active":"2024-09-21T10:22:15.056776+0000","last_peered":"2024-09-21T10:22:15.056776+0000","last_clean":"2024-09-21T10:21:59.002104+0000","last_became_active":"2024-09-21T10:22:01.317312+0000","last_became_peered":"2024-09-21T10:22:01.317312+0000","last_unstale":"2024-09-21T10:22:15.056776+0000","last_undegraded":"2024-09-21T10:22:00.997012+0000","last_fullsized":"2024-09-21T10:22:15.056776+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":637,"reported_epoch":351,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:15.049213+0000","last_change":"2024-09-21T10:21:34.866048+0000","last_active":"2024-09-21T10:22:15.049213+0000","last_peered":"2024-09-21T10:22:15.049213+0000","last_clean":"2024-09-21T10:21:17.726080+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:22:15.049213+0000","last_undegraded":"2024-09-21T10:21:34.856315+0000","last_fullsized":"2024-09-21T10:22:15.049213+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":324,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.051550+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:22:15.051550+0000","last_peered":"2024-09-21T10:22:15.051550+0000","last_clean":"2024-09-21T10:22:15.051550+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:22:15.051550+0000","last_undegraded":"2024-09-21T10:22:15.051550+0000","last_fullsized":"2024-09-21T10:22:15.051550+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":348,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.051858+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:22:15.051858+0000","last_peered":"2024-09-21T10:22:15.051858+0000","last_clean":"2024-09-21T10:22:15.051858+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:22:15.051858+0000","last_undegraded":"2024-09-21T10:22:15.051858+0000","last_fullsized":"2024-09-21T10:22:15.051858+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":58,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.051859+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:22:15.051859+0000","last_peered":"2024-09-21T10:22:15.051859+0000","last_clean":"2024-09-21T10:22:15.051859+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:22:15.051859+0000","last_undegraded":"2024-09-21T10:22:15.051859+0000","last_fullsized":"2024-09-21T10:22:15.051859+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":50,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.048736+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:22:15.048736+0000","last_peered":"2024-09-21T10:22:15.048736+0000","last_clean":"2024-09-21T10:22:15.048736+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:22:15.048736+0000","last_undegraded":"2024-09-21T10:22:15.048736+0000","last_fullsized":"2024-09-21T10:22:15.048736+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":30,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.065602+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:22:15.065602+0000","last_peered":"2024-09-21T10:22:15.065602+0000","last_clean":"2024-09-21T10:22:15.065602+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:22:15.065602+0000","last_undegraded":"2024-09-21T10:22:15.065602+0000","last_fullsized":"2024-09-21T10:22:15.065602+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":58,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.048709+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:22:15.048709+0000","last_peered":"2024-09-21T10:22:15.048709+0000","last_clean":"2024-09-21T10:22:15.048709+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:22:15.048709+0000","last_undegraded":"2024-09-21T10:22:15.048709+0000","last_fullsized":"2024-09-21T10:22:15.048709+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":533,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.051711+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:22:17.051711+0000","last_peered":"2024-09-21T10:22:17.051711+0000","last_clean":"2024-09-21T10:22:17.051711+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:22:17.051711+0000","last_undegraded":"2024-09-21T10:22:17.051711+0000","last_fullsized":"2024-09-21T10:22:17.051711+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":187,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.056955+0000","last_change":"2024-09-21T10:20:13.089473+0000","last_active":"2024-09-21T10:22:15.056955+0000","last_peered":"2024-09-21T10:22:15.056955+0000","last_clean":"2024-09-21T10:22:15.056955+0000","last_became_active":"2024-09-21T10:20:13.089111+0000","last_became_peered":"2024-09-21T10:20:13.089111+0000","last_unstale":"2024-09-21T10:22:15.056955+0000","last_undegraded":"2024-09-21T10:22:15.056955+0000","last_fullsized":"2024-09-21T10:22:15.056955+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:06.603284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":593,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052284+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:22:17.052284+0000","last_peered":"2024-09-21T10:22:17.052284+0000","last_clean":"2024-09-21T10:22:17.052284+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:22:17.052284+0000","last_undegraded":"2024-09-21T10:22:17.052284+0000","last_fullsized":"2024-09-21T10:22:17.052284+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":365,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.050256+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:22:15.050256+0000","last_peered":"2024-09-21T10:22:15.050256+0000","last_clean":"2024-09-21T10:22:15.050256+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:22:15.050256+0000","last_undegraded":"2024-09-21T10:22:15.050256+0000","last_fullsized":"2024-09-21T10:22:15.050256+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":610,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052455+0000","last_change":"2024-09-21T10:22:01.329690+0000","last_active":"2024-09-21T10:22:17.052455+0000","last_peered":"2024-09-21T10:22:17.052455+0000","last_clean":"2024-09-21T10:22:17.052455+0000","last_became_active":"2024-09-21T10:22:01.329033+0000","last_became_peered":"2024-09-21T10:22:01.329033+0000","last_unstale":"2024-09-21T10:22:17.052455+0000","last_undegraded":"2024-09-21T10:22:17.052455+0000","last_fullsized":"2024-09-21T10:22:17.052455+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:24.559524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":190,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.056990+0000","last_change":"2024-09-21T10:21:34.870704+0000","last_active":"2024-09-21T10:22:15.056990+0000","last_peered":"2024-09-21T10:22:15.056990+0000","last_clean":"2024-09-21T10:22:15.056990+0000","last_became_active":"2024-09-21T10:21:34.869835+0000","last_became_peered":"2024-09-21T10:21:34.869835+0000","last_unstale":"2024-09-21T10:22:15.056990+0000","last_undegraded":"2024-09-21T10:22:15.056990+0000","last_fullsized":"2024-09-21T10:22:15.056990+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:34.959915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052169+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:22:17.052169+0000","last_peered":"2024-09-21T10:22:17.052169+0000","last_clean":"2024-09-21T10:22:17.052169+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:22:17.052169+0000","last_undegraded":"2024-09-21T10:22:17.052169+0000","last_fullsized":"2024-09-21T10:22:17.052169+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":600,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054173+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:22:17.054173+0000","last_peered":"2024-09-21T10:22:17.054173+0000","last_clean":"2024-09-21T10:22:17.054173+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:22:17.054173+0000","last_undegraded":"2024-09-21T10:22:17.054173+0000","last_fullsized":"2024-09-21T10:22:17.054173+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":355,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.052308+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:22:15.052308+0000","last_peered":"2024-09-21T10:22:15.052308+0000","last_clean":"2024-09-21T10:22:15.052308+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:22:15.052308+0000","last_undegraded":"2024-09-21T10:22:15.052308+0000","last_fullsized":"2024-09-21T10:22:15.052308+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":599,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.059984+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:22:17.059984+0000","last_peered":"2024-09-21T10:22:17.059984+0000","last_clean":"2024-09-21T10:22:17.059984+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:22:17.059984+0000","last_undegraded":"2024-09-21T10:22:17.059984+0000","last_fullsized":"2024-09-21T10:22:17.059984+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":20,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.056492+0000","last_change":"2024-09-21T10:22:01.318549+0000","last_active":"2024-09-21T10:22:15.056492+0000","last_peered":"2024-09-21T10:22:15.056492+0000","last_clean":"2024-09-21T10:22:15.056492+0000","last_became_active":"2024-09-21T10:22:01.318071+0000","last_became_peered":"2024-09-21T10:22:01.318071+0000","last_unstale":"2024-09-21T10:22:15.056492+0000","last_undegraded":"2024-09-21T10:22:15.056492+0000","last_fullsized":"2024-09-21T10:22:15.056492+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:49.061492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":48,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053659+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:22:17.053659+0000","last_peered":"2024-09-21T10:22:17.053659+0000","last_clean":"2024-09-21T10:22:17.053659+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:22:17.053659+0000","last_undegraded":"2024-09-21T10:22:17.053659+0000","last_fullsized":"2024-09-21T10:22:17.053659+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":290,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.048691+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:22:15.048691+0000","last_peered":"2024-09-21T10:22:15.048691+0000","last_clean":"2024-09-21T10:22:15.048691+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:22:15.048691+0000","last_undegraded":"2024-09-21T10:22:15.048691+0000","last_fullsized":"2024-09-21T10:22:15.048691+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":48,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053324+0000","last_change":"2024-09-21T10:19:50.588363+0000","last_active":"2024-09-21T10:22:17.053324+0000","last_peered":"2024-09-21T10:22:17.053324+0000","last_clean":"2024-09-21T10:22:17.053324+0000","last_became_active":"2024-09-21T10:19:50.588107+0000","last_became_peered":"2024-09-21T10:19:50.588107+0000","last_unstale":"2024-09-21T10:22:17.053324+0000","last_undegraded":"2024-09-21T10:22:17.053324+0000","last_fullsized":"2024-09-21T10:22:17.053324+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:05.698018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":268,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.072912+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:22:15.072912+0000","last_peered":"2024-09-21T10:22:15.072912+0000","last_clean":"2024-09-21T10:22:15.072912+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:22:15.072912+0000","last_undegraded":"2024-09-21T10:22:15.072912+0000","last_fullsized":"2024-09-21T10:22:15.072912+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":613,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053821+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:22:17.053821+0000","last_peered":"2024-09-21T10:22:17.053821+0000","last_clean":"2024-09-21T10:22:17.053821+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:22:17.053821+0000","last_undegraded":"2024-09-21T10:22:17.053821+0000","last_fullsized":"2024-09-21T10:22:17.053821+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":575,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054029+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:22:17.054029+0000","last_peered":"2024-09-21T10:22:17.054029+0000","last_clean":"2024-09-21T10:22:17.054029+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:22:17.054029+0000","last_undegraded":"2024-09-21T10:22:17.054029+0000","last_fullsized":"2024-09-21T10:22:17.054029+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":48,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-21T10:22:15.051262+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:22:15.051262+0000","last_peered":"2024-09-21T10:22:15.051262+0000","last_clean":"2024-09-21T10:22:15.051262+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:22:15.051262+0000","last_undegraded":"2024-09-21T10:22:15.051262+0000","last_fullsized":"2024-09-21T10:22:15.051262+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":17,"reported_epoch":351,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:15.056881+0000","last_change":"2024-09-21T10:22:01.317167+0000","last_active":"2024-09-21T10:22:15.056881+0000","last_peered":"2024-09-21T10:22:15.056881+0000","last_clean":"2024-09-21T10:14:43.015500+0000","last_became_active":"2024-09-21T10:22:01.316569+0000","last_became_peered":"2024-09-21T10:22:01.316569+0000","last_unstale":"2024-09-21T10:22:15.056881+0000","last_undegraded":"2024-09-21T10:22:00.997069+0000","last_fullsized":"2024-09-21T10:22:15.056881+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4287823896,"num_objects":65428,"num_object_clones":0,"num_object_copies":196284,"num_objects_missing_on_primary":2897,"num_objects_missing":2897,"num_objects_degraded":21016,"num_objects_misplaced":10969,"num_objects_unfound":0,"num_objects_dirty":65428,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134258,"num_write_kb":6389889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15855,"num_bytes_recovered":1024393216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13004976128,"data_stored":13004963912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105929,"ondisk_log_size":105929,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":3,"num_objects_missing":3,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24225,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":391,"num_read_kb":833,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":3107188,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1640,"ondisk_log_size":1640,"up":33,"acting":31,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234392,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1384296,"kb_used_data":797792,"kb_used_omap":1,"kb_used_meta":586494,"kb_avail":92360856,"statfs":{"total":95995035648,"available":94577516544,"internally_reserved":0,"allocated":816939008,"data_stored":815144294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":600570315},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496030,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2647536,"kb_used_data":2042984,"kb_used_omap":1,"kb_used_meta":604542,"kb_avail":91097616,"statfs":{"total":95995035648,"available":93283958784,"internally_reserved":0,"allocated":2092015616,"data_stored":2090214064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":619051463},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403930,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1991460,"kb_used_data":1773272,"kb_used_omap":7,"kb_used_meta":218168,"kb_avail":91753692,"statfs":{"total":95995035648,"available":93955780608,"internally_reserved":0,"allocated":1815830528,"data_stored":1814040060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":223404952},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019303,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2156200,"kb_used_data":1579552,"kb_used_omap":1,"kb_used_meta":576638,"kb_avail":91588952,"statfs":{"total":95995035648,"available":93787086848,"internally_reserved":0,"allocated":1617461248,"data_stored":1615666795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":590477766},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435487,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2859608,"kb_used_data":2093516,"kb_used_omap":5,"kb_used_meta":766074,"kb_avail":90885544,"statfs":{"total":95995035648,"available":93066797056,"internally_reserved":0,"allocated":2143760384,"data_stored":2141981611,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":784459816},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599014,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2272544,"kb_used_data":1595284,"kb_used_omap":7,"kb_used_meta":677240,"kb_avail":91472608,"statfs":{"total":95995035648,"available":93667950592,"internally_reserved":0,"allocated":1633570816,"data_stored":1631784637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":693494221},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762541,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2433112,"kb_used_data":1833616,"kb_used_omap":7,"kb_used_meta":599480,"kb_avail":91312040,"statfs":{"total":95995035648,"available":93503528960,"internally_reserved":0,"allocated":1877622784,"data_stored":1875840137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":613867773},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926067,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1507144,"kb_used_data":1074236,"kb_used_omap":7,"kb_used_meta":432888,"kb_avail":92238008,"statfs":{"total":95995035648,"available":94451720192,"internally_reserved":0,"allocated":1100017664,"data_stored":1098246558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":443277502},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1086652416,"data_stored":1086652416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1861550080,"data_stored":1861550080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1618874368,"data_stored":1618870296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2133262336,"data_stored":2133262336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1609367552,"data_stored":1609367552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1810894848,"data_stored":1810890776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2081034240,"data_stored":2081030168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":803340288,"data_stored":803340288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:22:23.957 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:22:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.958 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:23.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:23 smithi045 ceph-mon[95976]: pgmap v1047: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 21041/195630 objects degraded (10.756%); 10969/195630 objects misplaced (5.607%); 12 MiB/s, 185 objects/s recovering 2024-09-21T10:22:24.059 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:24.085 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:24.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:24.187 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:24.217 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:24.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:24.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:24.344 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:24.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:24.445 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:24.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:24.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:24.572 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:24.602 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:24.704 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:24.726 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:24.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:24.828 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:24.850 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:24.952 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:24.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:24.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:25.081 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:25.110 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:25.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:25.211 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:25.245 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:25.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:25.372 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:25.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:25.443 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:22:25.444 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:22:25.474 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:25.499 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:25.601 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:25.631 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:25.662 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:25.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:25 smithi032 ceph-mon[101299]: pgmap v1048: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 20907/193782 objects degraded (10.789%); 10835/193782 objects misplaced (5.591%); 6.3 MiB/s, 101 objects/s recovering 2024-09-21T10:22:25.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:25 smithi032 ceph-mon[105748]: pgmap v1048: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 20907/193782 objects degraded (10.789%); 10835/193782 objects misplaced (5.591%); 6.3 MiB/s, 101 objects/s recovering 2024-09-21T10:22:25.733 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:25.762 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:25.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:25.864 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:25.889 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:25.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:25.990 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:26.023 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:26.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:25 smithi045 ceph-mon[95976]: pgmap v1048: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 20907/193782 objects degraded (10.789%); 10835/193782 objects misplaced (5.591%); 6.3 MiB/s, 101 objects/s recovering 2024-09-21T10:22:26.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:26.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:26.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:26.258 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:26.282 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:26.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:26.384 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:26.411 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:26.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:26.513 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:26.544 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:26.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:26.637 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:26.646 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:26.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.686 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:26.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:26.787 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:26.814 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:26.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:26.916 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:26.940 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:27.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:27.067 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:27.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:27.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:27.192 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:27.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:27.293 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:27.322 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:27.424 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:27.450 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:27.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:27.581 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:27.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:27.683 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:27.706 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:27.730 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:22:27.731 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:22:27.731 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:27 smithi032 ceph-mon[101299]: pgmap v1049: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 20307/190779 objects degraded (10.644%); 10371/190779 objects misplaced (5.436%); 8.9 MiB/s, 179 objects/s recovering 2024-09-21T10:22:27.731 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:27 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 20307/190779 objects degraded (10.644%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:27.731 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:27 smithi032 ceph-mon[105748]: pgmap v1049: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 20307/190779 objects degraded (10.644%); 10371/190779 objects misplaced (5.436%); 8.9 MiB/s, 179 objects/s recovering 2024-09-21T10:22:27.731 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:27 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 20307/190779 objects degraded (10.644%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:27.751 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:22:27.751 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:22:27.767 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:22:27.767 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:22:27.785 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:22:27.785 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:22:27.807 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:22:27.807 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:22:27.808 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:27.829 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:22:27.829 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:22:27.835 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:27.853 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:22:27.854 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:22:27.874 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:22:27.874 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:22:27.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:27.964 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:27.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:27 smithi045 ceph-mon[95976]: pgmap v1049: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 20307/190779 objects degraded (10.644%); 10371/190779 objects misplaced (5.436%); 8.9 MiB/s, 179 objects/s recovering 2024-09-21T10:22:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:27 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 20307/190779 objects degraded (10.644%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:28.065 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:28.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:28.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:28.193 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:28.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.225 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:28.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:28.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:28.356 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:28.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:28.458 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:28.493 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:28.574 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:28 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3534799178' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:28.574 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:28 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3534799178' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:28.594 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:28.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:28.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:28.726 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:28.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:28.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:28.886 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:28.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:28.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:29.011 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:29.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:28 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3534799178' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:29.113 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.142 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:29.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:29.243 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:29.271 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:29.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:29.373 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:29.400 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:29.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:29.503 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:29.527 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:29.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:29.555 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:22:29.629 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:29.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:29.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:29.758 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:29.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:29.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:29.920 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:29.958 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:22:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:22:29] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-21T10:22:29.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:29 smithi032 ceph-mon[101299]: pgmap v1050: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 316 op/s; 20307/190092 objects degraded (10.683%); 10371/190092 objects misplaced (5.456%); 7.8 MiB/s, 156 objects/s recovering 2024-09-21T10:22:29.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:29 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3253003600' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:29.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:29 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:29.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:29 smithi032 ceph-mon[105748]: pgmap v1050: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 316 op/s; 20307/190092 objects degraded (10.683%); 10371/190092 objects misplaced (5.456%); 7.8 MiB/s, 156 objects/s recovering 2024-09-21T10:22:29.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:29 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3253003600' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:29.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:29 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:30.022 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:29 smithi045 ceph-mon[95976]: pgmap v1050: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 316 op/s; 20307/190092 objects degraded (10.683%); 10371/190092 objects misplaced (5.456%); 7.8 MiB/s, 156 objects/s recovering 2024-09-21T10:22:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:29 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3253003600' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:29 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:30.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.057 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:30.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:30.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:30.187 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:30.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:30.289 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:30.323 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:30.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:30.425 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:30.452 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:30.554 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:30.580 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:30.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:30.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:30.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:30 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:30.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:30 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:22:30.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:30 smithi032 ceph-mon[101299]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T10:22:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:30 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:30 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:22:30.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:30 smithi032 ceph-mon[105748]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T10:22:30.710 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:30.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:30.812 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.846 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:30.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:30.932 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:22:30.947 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:31.005 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:31.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:30 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:30 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:22:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:30 smithi045 ceph-mon[95976]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T10:22:31.107 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:31.134 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:31.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:31.236 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:31.251 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:22:31.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:31.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:31.321 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:31.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:31.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.390 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:31.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:31.491 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:31.524 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:31.556 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:31 smithi032 ceph-mon[101299]: pgmap v1052: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 372 op/s; 19158/185199 objects degraded (10.345%); 10023/185199 objects misplaced (5.412%); 5.9 MiB/s, 211 objects/s recovering 2024-09-21T10:22:31.626 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:31.655 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:31.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:31.756 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:31.781 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:31.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:31.822 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:31 smithi032 ceph-mon[105748]: pgmap v1052: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 372 op/s; 19158/185199 objects degraded (10.345%); 10023/185199 objects misplaced (5.412%); 5.9 MiB/s, 211 objects/s recovering 2024-09-21T10:22:31.882 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:31.907 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:31.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:32.009 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:32.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:31 smithi045 ceph-mon[95976]: pgmap v1052: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 372 op/s; 19158/185199 objects degraded (10.345%); 10023/185199 objects misplaced (5.412%); 5.9 MiB/s, 211 objects/s recovering 2024-09-21T10:22:32.044 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:32.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:32.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:32.274 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:32.302 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:32.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:32.329 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:22:32.330 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:22:32.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:32.436 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:32.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:32.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:32.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:32.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:32.666 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:32.690 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:32.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:32.774 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:22:32.792 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:32.800 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:32 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 19158/185199 objects degraded (10.345%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:32.800 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:32 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 19158/185199 objects degraded (10.345%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:32.821 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:32.923 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:32.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:33.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:32 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 19158/185199 objects degraded (10.345%), 15 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:33.063 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:33.086 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:33.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:33.188 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:33.209 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1052,"stamp":"2024-09-21T10:22:30.519822+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4048623613,"num_objects":61733,"num_object_clones":0,"num_object_copies":185199,"num_objects_missing_on_primary":2682,"num_objects_missing":2682,"num_objects_degraded":19158,"num_objects_misplaced":10023,"num_objects_unfound":0,"num_objects_dirty":61733,"num_whiteouts":0,"num_read":6389,"num_read_kb":13844,"num_write":139104,"num_write_kb":6404930,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18246,"num_bytes_recovered":1106411513,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108625,"ondisk_log_size":108625,"up":297,"acting":288,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16641872,"kb_used_data":12172236,"kb_used_omap":40,"kb_used_meta":4469527,"kb_avail":733319344,"statfs":{"total":767960285184,"available":750919008256,"internally_reserved":0,"allocated":12464369664,"data_stored":12450054302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41824,"internal_metadata":4576795808},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-243990528,"num_objects":-3723,"num_object_clones":0,"num_object_copies":-11169,"num_objects_missing_on_primary":-220,"num_objects_missing":-220,"num_objects_degraded":-1883,"num_objects_misplaced":-946,"num_objects_unfound":0,"num_objects_dirty":-3723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3723,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2114,"num_bytes_recovered":62128128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002962"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":580,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053060+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:22:17.053060+0000","last_peered":"2024-09-21T10:22:17.053060+0000","last_clean":"2024-09-21T10:22:17.053060+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:22:17.053060+0000","last_undegraded":"2024-09-21T10:22:17.053060+0000","last_fullsized":"2024-09-21T10:22:17.053060+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":355,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067484+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:22:17.067484+0000","last_peered":"2024-09-21T10:22:17.067484+0000","last_clean":"2024-09-21T10:22:17.067484+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:22:17.067484+0000","last_undegraded":"2024-09-21T10:22:17.067484+0000","last_fullsized":"2024-09-21T10:22:17.067484+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":51,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067256+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:22:17.067256+0000","last_peered":"2024-09-21T10:22:17.067256+0000","last_clean":"2024-09-21T10:22:17.067256+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:22:17.067256+0000","last_undegraded":"2024-09-21T10:22:17.067256+0000","last_fullsized":"2024-09-21T10:22:17.067256+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":22,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067222+0000","last_change":"2024-09-21T10:22:01.318638+0000","last_active":"2024-09-21T10:22:17.067222+0000","last_peered":"2024-09-21T10:22:17.067222+0000","last_clean":"2024-09-21T10:22:17.067222+0000","last_became_active":"2024-09-21T10:22:01.317087+0000","last_became_peered":"2024-09-21T10:22:01.317087+0000","last_unstale":"2024-09-21T10:22:17.067222+0000","last_undegraded":"2024-09-21T10:22:17.067222+0000","last_fullsized":"2024-09-21T10:22:17.067222+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:25.334726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":23,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.066986+0000","last_change":"2024-09-21T10:22:01.318978+0000","last_active":"2024-09-21T10:22:17.066986+0000","last_peered":"2024-09-21T10:22:17.066986+0000","last_clean":"2024-09-21T10:22:17.066986+0000","last_became_active":"2024-09-21T10:22:01.318255+0000","last_became_peered":"2024-09-21T10:22:01.318255+0000","last_unstale":"2024-09-21T10:22:17.066986+0000","last_undegraded":"2024-09-21T10:22:17.066986+0000","last_fullsized":"2024-09-21T10:22:17.066986+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:55.084088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":326,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067579+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:22:17.067579+0000","last_peered":"2024-09-21T10:22:17.067579+0000","last_clean":"2024-09-21T10:22:17.067579+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:22:17.067579+0000","last_undegraded":"2024-09-21T10:22:17.067579+0000","last_fullsized":"2024-09-21T10:22:17.067579+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":601,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054082+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:22:17.054082+0000","last_peered":"2024-09-21T10:22:17.054082+0000","last_clean":"2024-09-21T10:22:17.054082+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:22:17.054082+0000","last_undegraded":"2024-09-21T10:22:17.054082+0000","last_fullsized":"2024-09-21T10:22:17.054082+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":407,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.055560+0000","last_change":"2024-09-21T10:22:01.331612+0000","last_active":"2024-09-21T10:22:17.055560+0000","last_peered":"2024-09-21T10:22:17.055560+0000","last_clean":"2024-09-21T10:22:17.055560+0000","last_became_active":"2024-09-21T10:22:01.325282+0000","last_became_peered":"2024-09-21T10:22:01.325282+0000","last_unstale":"2024-09-21T10:22:17.055560+0000","last_undegraded":"2024-09-21T10:22:17.055560+0000","last_fullsized":"2024-09-21T10:22:17.055560+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:23.383106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"353'8677","reported_seq":9737,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:29.948063+0000","last_change":"2024-09-21T10:21:39.510930+0000","last_active":"2024-09-21T10:22:29.948063+0000","last_peered":"2024-09-21T10:22:29.948063+0000","last_clean":"2024-09-21T10:20:12.056605+0000","last_became_active":"2024-09-21T10:20:14.466147+0000","last_became_peered":"2024-09-21T10:20:14.466147+0000","last_unstale":"2024-09-21T10:22:29.948063+0000","last_undegraded":"2024-09-21T10:20:14.094800+0000","last_fullsized":"2024-09-21T10:20:14.071992+0000","mapping_epoch":325,"log_start":"314'1300","ondisk_log_start":"314'1300","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7377,"log_dups_size":0,"ondisk_log_size":7377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:34.237367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251854848,"num_objects":3843,"num_object_clones":0,"num_object_copies":11529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3167,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8677,"num_write_kb":400640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":489,"num_bytes_recovered":32047104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":676},{"shards":"2,4,6","objects":1238}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":42,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.068100+0000","last_change":"2024-09-21T10:22:01.321467+0000","last_active":"2024-09-21T10:22:17.068100+0000","last_peered":"2024-09-21T10:22:17.068100+0000","last_clean":"2024-09-21T10:22:17.068100+0000","last_became_active":"2024-09-21T10:22:01.318270+0000","last_became_peered":"2024-09-21T10:22:01.318270+0000","last_unstale":"2024-09-21T10:22:17.068100+0000","last_undegraded":"2024-09-21T10:22:17.068100+0000","last_fullsized":"2024-09-21T10:22:17.068100+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:44.059553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:19.654269+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:22:19.654269+0000","last_peered":"2024-09-21T10:22:19.654269+0000","last_clean":"2024-09-21T10:22:19.654269+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:22:19.654269+0000","last_undegraded":"2024-09-21T10:22:19.654269+0000","last_fullsized":"2024-09-21T10:22:19.654269+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"353'8454","reported_seq":12322,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.924203+0000","last_change":"2024-09-21T10:20:04.684629+0000","last_active":"2024-09-21T10:22:29.924203+0000","last_peered":"2024-09-21T10:22:29.924203+0000","last_clean":"2024-09-21T10:22:29.924203+0000","last_became_active":"2024-09-21T10:20:04.680299+0000","last_became_peered":"2024-09-21T10:20:04.680299+0000","last_unstale":"2024-09-21T10:22:29.924203+0000","last_undegraded":"2024-09-21T10:22:29.924203+0000","last_fullsized":"2024-09-21T10:22:29.924203+0000","mapping_epoch":320,"log_start":"317'1900","ondisk_log_start":"317'1900","created":305,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:41.743081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248774656,"num_objects":3796,"num_object_clones":0,"num_object_copies":11388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8454,"num_write_kb":392000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1891,"num_bytes_recovered":123928576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":29,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067774+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:22:17.067774+0000","last_peered":"2024-09-21T10:22:17.067774+0000","last_clean":"2024-09-21T10:22:17.067774+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:22:17.067774+0000","last_undegraded":"2024-09-21T10:22:17.067774+0000","last_fullsized":"2024-09-21T10:22:17.067774+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":49,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053226+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:22:17.053226+0000","last_peered":"2024-09-21T10:22:17.053226+0000","last_clean":"2024-09-21T10:22:17.053226+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:22:17.053226+0000","last_undegraded":"2024-09-21T10:22:17.053226+0000","last_fullsized":"2024-09-21T10:22:17.053226+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"352'8412","reported_seq":8518,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:26.576069+0000","last_change":"2024-09-21T10:19:22.649712+0000","last_active":"2024-09-21T10:22:26.576069+0000","last_peered":"2024-09-21T10:22:26.576069+0000","last_clean":"2024-09-21T10:22:26.576069+0000","last_became_active":"2024-09-21T10:19:22.649303+0000","last_became_peered":"2024-09-21T10:19:22.649303+0000","last_unstale":"2024-09-21T10:22:26.576069+0000","last_undegraded":"2024-09-21T10:22:26.576069+0000","last_fullsized":"2024-09-21T10:22:26.576069+0000","mapping_epoch":305,"log_start":"314'1261","ondisk_log_start":"314'1261","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7151,"log_dups_size":0,"ondisk_log_size":7151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:24.442851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255066112,"num_objects":3892,"num_object_clones":0,"num_object_copies":11676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8412,"num_write_kb":393728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":351,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.068066+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:22:17.068066+0000","last_peered":"2024-09-21T10:22:17.068066+0000","last_clean":"2024-09-21T10:22:17.068066+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:22:17.068066+0000","last_undegraded":"2024-09-21T10:22:17.068066+0000","last_fullsized":"2024-09-21T10:22:17.068066+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":370,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.055570+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:22:17.055570+0000","last_peered":"2024-09-21T10:22:17.055570+0000","last_clean":"2024-09-21T10:22:17.055570+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:22:17.055570+0000","last_undegraded":"2024-09-21T10:22:17.055570+0000","last_fullsized":"2024-09-21T10:22:17.055570+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"353'8884","reported_seq":8992,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:30.284038+0000","last_change":"2024-09-21T10:19:22.640645+0000","last_active":"2024-09-21T10:22:30.284038+0000","last_peered":"2024-09-21T10:22:30.284038+0000","last_clean":"2024-09-21T10:22:30.284038+0000","last_became_active":"2024-09-21T10:19:22.640343+0000","last_became_peered":"2024-09-21T10:19:22.640343+0000","last_unstale":"2024-09-21T10:22:30.284038+0000","last_undegraded":"2024-09-21T10:22:30.284038+0000","last_fullsized":"2024-09-21T10:22:30.284038+0000","mapping_epoch":305,"log_start":"322'3600","ondisk_log_start":"322'3600","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5284,"log_dups_size":0,"ondisk_log_size":5284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:48.734122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259391488,"num_objects":3958,"num_object_clones":0,"num_object_copies":11874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8884,"num_write_kb":410944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":285,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:24.665369+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:22:24.665369+0000","last_peered":"2024-09-21T10:22:24.665369+0000","last_clean":"2024-09-21T10:22:24.665369+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:22:24.665369+0000","last_undegraded":"2024-09-21T10:22:24.665369+0000","last_fullsized":"2024-09-21T10:22:24.665369+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":192,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.066787+0000","last_change":"2024-09-21T10:19:50.585724+0000","last_active":"2024-09-21T10:22:17.066787+0000","last_peered":"2024-09-21T10:22:17.066787+0000","last_clean":"2024-09-21T10:22:17.066787+0000","last_became_active":"2024-09-21T10:19:50.585315+0000","last_became_peered":"2024-09-21T10:19:50.585315+0000","last_unstale":"2024-09-21T10:22:17.066787+0000","last_undegraded":"2024-09-21T10:22:17.066787+0000","last_fullsized":"2024-09-21T10:22:17.066787+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:26.618970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"352'8630","reported_seq":13699,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:28.785008+0000","last_change":"2024-09-21T10:22:01.328242+0000","last_active":"2024-09-21T10:22:28.785008+0000","last_peered":"2024-09-21T10:22:28.785008+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492359+0000","last_became_peered":"2024-09-21T10:20:14.492359+0000","last_unstale":"2024-09-21T10:22:28.785008+0000","last_undegraded":"2024-09-21T10:20:14.218670+0000","last_fullsized":"2024-09-21T10:20:14.071510+0000","mapping_epoch":325,"log_start":"314'1800","ondisk_log_start":"314'1800","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6830,"log_dups_size":0,"ondisk_log_size":6830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255197184,"num_objects":3894,"num_object_clones":0,"num_object_copies":11682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8630,"num_write_kb":400768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2481,"num_bytes_recovered":148307968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":2690}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":607,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.066715+0000","last_change":"2024-09-21T10:22:04.029688+0000","last_active":"2024-09-21T10:22:17.066715+0000","last_peered":"2024-09-21T10:22:17.066715+0000","last_clean":"2024-09-21T10:22:17.066715+0000","last_became_active":"2024-09-21T10:22:04.028853+0000","last_became_peered":"2024-09-21T10:22:04.028853+0000","last_unstale":"2024-09-21T10:22:17.066715+0000","last_undegraded":"2024-09-21T10:22:17.066715+0000","last_fullsized":"2024-09-21T10:22:17.066715+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:20.457596+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:19:20.457596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":631,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054741+0000","last_change":"2024-09-21T10:22:01.335303+0000","last_active":"2024-09-21T10:22:17.054741+0000","last_peered":"2024-09-21T10:22:17.054741+0000","last_clean":"2024-09-21T10:22:17.054741+0000","last_became_active":"2024-09-21T10:22:01.325298+0000","last_became_peered":"2024-09-21T10:22:01.325298+0000","last_unstale":"2024-09-21T10:22:17.054741+0000","last_undegraded":"2024-09-21T10:22:17.054741+0000","last_fullsized":"2024-09-21T10:22:17.054741+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:01.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":49,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053547+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:22:17.053547+0000","last_peered":"2024-09-21T10:22:17.053547+0000","last_clean":"2024-09-21T10:22:17.053547+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:22:17.053547+0000","last_undegraded":"2024-09-21T10:22:17.053547+0000","last_fullsized":"2024-09-21T10:22:17.053547+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"353'8505","reported_seq":8613,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:30.228257+0000","last_change":"2024-09-21T10:19:22.641281+0000","last_active":"2024-09-21T10:22:30.228257+0000","last_peered":"2024-09-21T10:22:30.228257+0000","last_clean":"2024-09-21T10:22:30.228257+0000","last_became_active":"2024-09-21T10:19:22.640988+0000","last_became_peered":"2024-09-21T10:19:22.640988+0000","last_unstale":"2024-09-21T10:22:30.228257+0000","last_undegraded":"2024-09-21T10:22:30.228257+0000","last_fullsized":"2024-09-21T10:22:30.228257+0000","mapping_epoch":305,"log_start":"321'3200","ondisk_log_start":"321'3200","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:52.652565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242286592,"num_objects":3697,"num_object_clones":0,"num_object_copies":11091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8505,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":421,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:17.066874+0000","last_change":"2024-09-21T10:22:02.013601+0000","last_active":"2024-09-21T10:22:17.066874+0000","last_peered":"2024-09-21T10:22:17.066874+0000","last_clean":"2024-09-21T10:19:15.364615+0000","last_became_active":"2024-09-21T10:22:02.013140+0000","last_became_peered":"2024-09-21T10:22:02.013140+0000","last_unstale":"2024-09-21T10:22:17.066874+0000","last_undegraded":"2024-09-21T10:22:02.004849+0000","last_fullsized":"2024-09-21T10:22:02.002552+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_clean_scrub_stamp":"2024-09-21T10:19:15.365774+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1},{"shards":"6,7","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":360,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.055940+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:22:17.055940+0000","last_peered":"2024-09-21T10:22:17.055940+0000","last_clean":"2024-09-21T10:22:17.055940+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:22:17.055940+0000","last_undegraded":"2024-09-21T10:22:17.055940+0000","last_fullsized":"2024-09-21T10:22:17.055940+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":49,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053766+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:22:17.053766+0000","last_peered":"2024-09-21T10:22:17.053766+0000","last_clean":"2024-09-21T10:22:17.053766+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:22:17.053766+0000","last_undegraded":"2024-09-21T10:22:17.053766+0000","last_fullsized":"2024-09-21T10:22:17.053766+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"353'8692","reported_seq":14814,"reported_epoch":353,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:29.976132+0000","last_change":"2024-09-21T10:22:01.331955+0000","last_active":"2024-09-21T10:22:29.976132+0000","last_peered":"2024-09-21T10:22:29.976132+0000","last_clean":"2024-09-21T10:20:12.057304+0000","last_became_active":"2024-09-21T10:20:14.547332+0000","last_became_peered":"2024-09-21T10:20:14.547332+0000","last_unstale":"2024-09-21T10:22:29.976132+0000","last_undegraded":"2024-09-21T10:20:14.094983+0000","last_fullsized":"2024-09-21T10:20:14.073184+0000","mapping_epoch":325,"log_start":"314'1300","ondisk_log_start":"314'1300","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7392,"log_dups_size":0,"ondisk_log_size":7392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:10.148038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257032192,"num_objects":3922,"num_object_clones":0,"num_object_copies":11766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":596,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8692,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":121634816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,4","objects":3326}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":49,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053069+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:22:17.053069+0000","last_peered":"2024-09-21T10:22:17.053069+0000","last_clean":"2024-09-21T10:22:17.053069+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:22:17.053069+0000","last_undegraded":"2024-09-21T10:22:17.053069+0000","last_fullsized":"2024-09-21T10:22:17.053069+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":49,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053944+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:22:17.053944+0000","last_peered":"2024-09-21T10:22:17.053944+0000","last_clean":"2024-09-21T10:22:17.053944+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:22:17.053944+0000","last_undegraded":"2024-09-21T10:22:17.053944+0000","last_fullsized":"2024-09-21T10:22:17.053944+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"353'8644","reported_seq":8765,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:30.335812+0000","last_change":"2024-09-21T10:21:39.558048+0000","last_active":"2024-09-21T10:22:30.335812+0000","last_peered":"2024-09-21T10:22:30.335812+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492538+0000","last_became_peered":"2024-09-21T10:20:14.492538+0000","last_unstale":"2024-09-21T10:22:30.335812+0000","last_undegraded":"2024-09-21T10:20:14.216051+0000","last_fullsized":"2024-09-21T10:20:14.075057+0000","mapping_epoch":325,"log_start":"322'3300","ondisk_log_start":"322'3300","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5344,"log_dups_size":0,"ondisk_log_size":5344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254148608,"num_objects":3878,"num_object_clones":0,"num_object_copies":11634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8644,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":302}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":607,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067144+0000","last_change":"2024-09-21T10:22:04.031320+0000","last_active":"2024-09-21T10:22:17.067144+0000","last_peered":"2024-09-21T10:22:17.067144+0000","last_clean":"2024-09-21T10:22:17.067144+0000","last_became_active":"2024-09-21T10:22:04.030774+0000","last_became_peered":"2024-09-21T10:22:04.030774+0000","last_unstale":"2024-09-21T10:22:17.067144+0000","last_undegraded":"2024-09-21T10:22:17.067144+0000","last_fullsized":"2024-09-21T10:22:17.067144+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:22.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:22.489745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":609,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052533+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:22:17.052533+0000","last_peered":"2024-09-21T10:22:17.052533+0000","last_clean":"2024-09-21T10:22:17.052533+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:22:17.052533+0000","last_undegraded":"2024-09-21T10:22:17.052533+0000","last_fullsized":"2024-09-21T10:22:17.052533+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":304,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.620076+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:22:29.620076+0000","last_peered":"2024-09-21T10:22:29.620076+0000","last_clean":"2024-09-21T10:22:29.620076+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:22:29.620076+0000","last_undegraded":"2024-09-21T10:22:29.620076+0000","last_fullsized":"2024-09-21T10:22:29.620076+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"353'8648","reported_seq":9992,"reported_epoch":353,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:22:29.933673+0000","last_change":"2024-09-21T10:22:02.106012+0000","last_active":"2024-09-21T10:22:29.933673+0000","last_peered":"2024-09-21T10:22:29.933673+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:22:02.086281+0000","last_became_peered":"2024-09-21T10:22:02.086281+0000","last_unstale":"2024-09-21T10:22:29.933673+0000","last_undegraded":"2024-09-21T10:22:02.081791+0000","last_fullsized":"2024-09-21T10:22:29.933673+0000","mapping_epoch":347,"log_start":"318'2100","ondisk_log_start":"318'2100","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:37.553227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248643584,"num_objects":3794,"num_object_clones":0,"num_object_copies":11382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3037,"num_objects_misplaced":3794,"num_objects_unfound":0,"num_objects_dirty":3794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8648,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,5,6","objects":2550},{"shards":"3,6","objects":1244}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":607,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.066763+0000","last_change":"2024-09-21T10:22:04.029284+0000","last_active":"2024-09-21T10:22:17.066763+0000","last_peered":"2024-09-21T10:22:17.066763+0000","last_clean":"2024-09-21T10:22:17.066763+0000","last_became_active":"2024-09-21T10:22:04.028938+0000","last_became_peered":"2024-09-21T10:22:04.028938+0000","last_unstale":"2024-09-21T10:22:17.066763+0000","last_undegraded":"2024-09-21T10:22:17.066763+0000","last_fullsized":"2024-09-21T10:22:17.066763+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:12.388304+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:12.388304+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":637,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:17.051399+0000","last_change":"2024-09-21T10:21:36.010162+0000","last_active":"2024-09-21T10:22:17.051399+0000","last_peered":"2024-09-21T10:22:17.051399+0000","last_clean":"2024-09-21T10:21:17.725138+0000","last_became_active":"2024-09-21T10:21:36.009706+0000","last_became_peered":"2024-09-21T10:21:36.009706+0000","last_unstale":"2024-09-21T10:22:17.051399+0000","last_undegraded":"2024-09-21T10:21:35.852171+0000","last_fullsized":"2024-09-21T10:21:35.851526+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:04.206298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":540,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053470+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:22:17.053470+0000","last_peered":"2024-09-21T10:22:17.053470+0000","last_clean":"2024-09-21T10:22:17.053470+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:22:17.053470+0000","last_undegraded":"2024-09-21T10:22:17.053470+0000","last_fullsized":"2024-09-21T10:22:17.053470+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":327,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.056327+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:22:17.056327+0000","last_peered":"2024-09-21T10:22:17.056327+0000","last_clean":"2024-09-21T10:22:17.056327+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:22:17.056327+0000","last_undegraded":"2024-09-21T10:22:17.056327+0000","last_fullsized":"2024-09-21T10:22:17.056327+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"353'8470","reported_seq":8578,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:30.366185+0000","last_change":"2024-09-21T10:20:14.480108+0000","last_active":"2024-09-21T10:22:30.366185+0000","last_peered":"2024-09-21T10:22:30.366185+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.471864+0000","last_became_peered":"2024-09-21T10:20:14.471864+0000","last_unstale":"2024-09-21T10:22:30.366185+0000","last_undegraded":"2024-09-21T10:20:14.214064+0000","last_fullsized":"2024-09-21T10:20:14.074889+0000","mapping_epoch":325,"log_start":"321'3200","ondisk_log_start":"321'3200","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5270,"log_dups_size":0,"ondisk_log_size":5270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244711424,"num_objects":3734,"num_object_clones":0,"num_object_copies":11202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8470,"num_write_kb":390528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":189}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":607,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067285+0000","last_change":"2024-09-21T10:22:04.031573+0000","last_active":"2024-09-21T10:22:17.067285+0000","last_peered":"2024-09-21T10:22:17.067285+0000","last_clean":"2024-09-21T10:22:17.067285+0000","last_became_active":"2024-09-21T10:22:04.031125+0000","last_became_peered":"2024-09-21T10:22:04.031125+0000","last_unstale":"2024-09-21T10:22:17.067285+0000","last_undegraded":"2024-09-21T10:22:17.067285+0000","last_fullsized":"2024-09-21T10:22:17.067285+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:19:19.450903+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":992,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067099+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:22:17.067099+0000","last_peered":"2024-09-21T10:22:17.067099+0000","last_clean":"2024-09-21T10:22:17.067099+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:22:17.067099+0000","last_undegraded":"2024-09-21T10:22:17.067099+0000","last_fullsized":"2024-09-21T10:22:17.067099+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":190,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.066818+0000","last_change":"2024-09-21T10:19:51.555124+0000","last_active":"2024-09-21T10:22:17.066818+0000","last_peered":"2024-09-21T10:22:17.066818+0000","last_clean":"2024-09-21T10:22:17.066818+0000","last_became_active":"2024-09-21T10:19:51.554766+0000","last_became_peered":"2024-09-21T10:19:51.554766+0000","last_unstale":"2024-09-21T10:22:17.066818+0000","last_undegraded":"2024-09-21T10:22:17.066818+0000","last_fullsized":"2024-09-21T10:22:17.066818+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:52.832950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":22,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.066629+0000","last_change":"2024-09-21T10:22:01.316140+0000","last_active":"2024-09-21T10:22:17.066629+0000","last_peered":"2024-09-21T10:22:17.066629+0000","last_clean":"2024-09-21T10:22:17.066629+0000","last_became_active":"2024-09-21T10:22:01.315408+0000","last_became_peered":"2024-09-21T10:22:01.315408+0000","last_unstale":"2024-09-21T10:22:17.066629+0000","last_undegraded":"2024-09-21T10:22:17.066629+0000","last_fullsized":"2024-09-21T10:22:17.066629+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:07.322279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"352'8660","reported_seq":12627,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:28.761218+0000","last_change":"2024-09-21T10:20:47.589372+0000","last_active":"2024-09-21T10:22:28.761218+0000","last_peered":"2024-09-21T10:22:28.761218+0000","last_clean":"2024-09-21T10:22:28.761218+0000","last_became_active":"2024-09-21T10:20:47.580051+0000","last_became_peered":"2024-09-21T10:20:47.580051+0000","last_unstale":"2024-09-21T10:22:28.761218+0000","last_undegraded":"2024-09-21T10:22:28.761218+0000","last_fullsized":"2024-09-21T10:22:28.761218+0000","mapping_epoch":332,"log_start":"314'1800","ondisk_log_start":"314'1800","created":305,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6860,"log_dups_size":0,"ondisk_log_size":6860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:46.363318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258080768,"num_objects":3938,"num_object_clones":0,"num_object_copies":11814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8660,"num_write_kb":403136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":607,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067071+0000","last_change":"2024-09-21T10:22:04.030341+0000","last_active":"2024-09-21T10:22:17.067071+0000","last_peered":"2024-09-21T10:22:17.067071+0000","last_clean":"2024-09-21T10:22:17.067071+0000","last_became_active":"2024-09-21T10:22:04.030000+0000","last_became_peered":"2024-09-21T10:22:04.030000+0000","last_unstale":"2024-09-21T10:22:17.067071+0000","last_undegraded":"2024-09-21T10:22:17.067071+0000","last_fullsized":"2024-09-21T10:22:17.067071+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:10.332770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:10.332770+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1130,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.059659+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:22:17.059659+0000","last_peered":"2024-09-21T10:22:17.059659+0000","last_clean":"2024-09-21T10:22:17.059659+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:22:17.059659+0000","last_undegraded":"2024-09-21T10:22:17.059659+0000","last_fullsized":"2024-09-21T10:22:17.059659+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":290,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.621628+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:22:29.621628+0000","last_peered":"2024-09-21T10:22:29.621628+0000","last_clean":"2024-09-21T10:22:29.621628+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:22:29.621628+0000","last_undegraded":"2024-09-21T10:22:29.621628+0000","last_fullsized":"2024-09-21T10:22:29.621628+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":320,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.620116+0000","last_change":"2024-09-21T10:22:01.317933+0000","last_active":"2024-09-21T10:22:29.620116+0000","last_peered":"2024-09-21T10:22:29.620116+0000","last_clean":"2024-09-21T10:22:29.620116+0000","last_became_active":"2024-09-21T10:22:01.317536+0000","last_became_peered":"2024-09-21T10:22:01.317536+0000","last_unstale":"2024-09-21T10:22:29.620116+0000","last_undegraded":"2024-09-21T10:22:29.620116+0000","last_fullsized":"2024-09-21T10:22:29.620116+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:34.312724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"352'8666","reported_seq":15858,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:28.779024+0000","last_change":"2024-09-21T10:21:34.868559+0000","last_active":"2024-09-21T10:22:28.779024+0000","last_peered":"2024-09-21T10:22:28.779024+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.463996+0000","last_became_peered":"2024-09-21T10:20:14.463996+0000","last_unstale":"2024-09-21T10:22:28.779024+0000","last_undegraded":"2024-09-21T10:20:14.217266+0000","last_fullsized":"2024-09-21T10:20:14.071353+0000","mapping_epoch":325,"log_start":"314'1800","ondisk_log_start":"314'1800","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6866,"log_dups_size":0,"ondisk_log_size":6866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262406144,"num_objects":4004,"num_object_clones":0,"num_object_copies":12012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":98,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8666,"num_write_kb":405440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":232128512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3906}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":607,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067452+0000","last_change":"2024-09-21T10:22:04.029892+0000","last_active":"2024-09-21T10:22:17.067452+0000","last_peered":"2024-09-21T10:22:17.067452+0000","last_clean":"2024-09-21T10:22:17.067452+0000","last_became_active":"2024-09-21T10:22:04.029353+0000","last_became_peered":"2024-09-21T10:22:04.029353+0000","last_unstale":"2024-09-21T10:22:17.067452+0000","last_undegraded":"2024-09-21T10:22:17.067452+0000","last_fullsized":"2024-09-21T10:22:17.067452+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:19:13.371825+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1095,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052790+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:22:17.052790+0000","last_peered":"2024-09-21T10:22:17.052790+0000","last_clean":"2024-09-21T10:22:17.052790+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:22:17.052790+0000","last_undegraded":"2024-09-21T10:22:17.052790+0000","last_fullsized":"2024-09-21T10:22:17.052790+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":49,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053799+0000","last_change":"2024-09-21T10:19:50.581529+0000","last_active":"2024-09-21T10:22:17.053799+0000","last_peered":"2024-09-21T10:22:17.053799+0000","last_clean":"2024-09-21T10:22:17.053799+0000","last_became_active":"2024-09-21T10:19:50.581119+0000","last_became_peered":"2024-09-21T10:19:50.581119+0000","last_unstale":"2024-09-21T10:22:17.053799+0000","last_undegraded":"2024-09-21T10:22:17.053799+0000","last_fullsized":"2024-09-21T10:22:17.053799+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:13.334483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":290,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.619552+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:22:29.619552+0000","last_peered":"2024-09-21T10:22:29.619552+0000","last_clean":"2024-09-21T10:22:29.619552+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:22:29.619552+0000","last_undegraded":"2024-09-21T10:22:29.619552+0000","last_fullsized":"2024-09-21T10:22:29.619552+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"352'8767","reported_seq":12862,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:26.578909+0000","last_change":"2024-09-21T10:20:42.601607+0000","last_active":"2024-09-21T10:22:26.578909+0000","last_peered":"2024-09-21T10:22:26.578909+0000","last_clean":"2024-09-21T10:22:26.578909+0000","last_became_active":"2024-09-21T10:20:42.593406+0000","last_became_peered":"2024-09-21T10:20:42.593406+0000","last_unstale":"2024-09-21T10:22:26.578909+0000","last_undegraded":"2024-09-21T10:22:26.578909+0000","last_fullsized":"2024-09-21T10:22:26.578909+0000","mapping_epoch":330,"log_start":"314'1568","ondisk_log_start":"314'1568","created":305,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7199,"log_dups_size":0,"ondisk_log_size":7199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:09.770489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259194880,"num_objects":3955,"num_object_clones":0,"num_object_copies":11865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8767,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1985,"num_bytes_recovered":130088960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":607,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.068215+0000","last_change":"2024-09-21T10:22:04.029371+0000","last_active":"2024-09-21T10:22:17.068215+0000","last_peered":"2024-09-21T10:22:17.068215+0000","last_clean":"2024-09-21T10:22:17.068215+0000","last_became_active":"2024-09-21T10:22:04.029014+0000","last_became_peered":"2024-09-21T10:22:04.029014+0000","last_unstale":"2024-09-21T10:22:17.068215+0000","last_undegraded":"2024-09-21T10:22:17.068215+0000","last_fullsized":"2024-09-21T10:22:17.068215+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":300,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:19.658949+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:22:19.658949+0000","last_peered":"2024-09-21T10:22:19.658949+0000","last_clean":"2024-09-21T10:22:19.658949+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:22:19.658949+0000","last_undegraded":"2024-09-21T10:22:19.658949+0000","last_fullsized":"2024-09-21T10:22:19.658949+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":899,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.619777+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:22:29.619777+0000","last_peered":"2024-09-21T10:22:29.619777+0000","last_clean":"2024-09-21T10:22:29.619777+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:22:29.619777+0000","last_undegraded":"2024-09-21T10:22:29.619777+0000","last_fullsized":"2024-09-21T10:22:29.619777+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":326,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.619555+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:22:29.619555+0000","last_peered":"2024-09-21T10:22:29.619555+0000","last_clean":"2024-09-21T10:22:29.619555+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:22:29.619555+0000","last_undegraded":"2024-09-21T10:22:29.619555+0000","last_fullsized":"2024-09-21T10:22:29.619555+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"352'8549","reported_seq":8655,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:28.771243+0000","last_change":"2024-09-21T10:19:22.641043+0000","last_active":"2024-09-21T10:22:28.771243+0000","last_peered":"2024-09-21T10:22:28.771243+0000","last_clean":"2024-09-21T10:22:28.771243+0000","last_became_active":"2024-09-21T10:19:22.640742+0000","last_became_peered":"2024-09-21T10:19:22.640742+0000","last_unstale":"2024-09-21T10:22:28.771243+0000","last_undegraded":"2024-09-21T10:22:28.771243+0000","last_fullsized":"2024-09-21T10:22:28.771243+0000","mapping_epoch":305,"log_start":"314'1700","ondisk_log_start":"314'1700","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6849,"log_dups_size":0,"ondisk_log_size":6849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:36.901208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248840192,"num_objects":3797,"num_object_clones":0,"num_object_copies":11391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8549,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":607,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067846+0000","last_change":"2024-09-21T10:22:04.031872+0000","last_active":"2024-09-21T10:22:17.067846+0000","last_peered":"2024-09-21T10:22:17.067846+0000","last_clean":"2024-09-21T10:22:17.067846+0000","last_became_active":"2024-09-21T10:22:04.031343+0000","last_became_peered":"2024-09-21T10:22:04.031343+0000","last_unstale":"2024-09-21T10:22:17.067846+0000","last_undegraded":"2024-09-21T10:22:17.067846+0000","last_fullsized":"2024-09-21T10:22:17.067846+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:07.332404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:07.332404+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":308,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.619693+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:22:29.619693+0000","last_peered":"2024-09-21T10:22:29.619693+0000","last_clean":"2024-09-21T10:22:29.619693+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:22:29.619693+0000","last_undegraded":"2024-09-21T10:22:29.619693+0000","last_fullsized":"2024-09-21T10:22:29.619693+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":918,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.068141+0000","last_change":"2024-09-21T10:22:01.336961+0000","last_active":"2024-09-21T10:22:17.068141+0000","last_peered":"2024-09-21T10:22:17.068141+0000","last_clean":"2024-09-21T10:22:17.068141+0000","last_became_active":"2024-09-21T10:22:01.320079+0000","last_became_peered":"2024-09-21T10:22:01.320079+0000","last_unstale":"2024-09-21T10:22:17.068141+0000","last_undegraded":"2024-09-21T10:22:17.068141+0000","last_fullsized":"2024-09-21T10:22:17.068141+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:54.037388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":189,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.066636+0000","last_change":"2024-09-21T10:20:13.088557+0000","last_active":"2024-09-21T10:22:17.066636+0000","last_peered":"2024-09-21T10:22:17.066636+0000","last_clean":"2024-09-21T10:22:17.066636+0000","last_became_active":"2024-09-21T10:20:13.088123+0000","last_became_peered":"2024-09-21T10:20:13.088123+0000","last_unstale":"2024-09-21T10:22:17.066636+0000","last_undegraded":"2024-09-21T10:22:17.066636+0000","last_fullsized":"2024-09-21T10:22:17.066636+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:49.475499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"352'8723","reported_seq":10917,"reported_epoch":352,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:22:28.835196+0000","last_change":"2024-09-21T10:22:01.405956+0000","last_active":"2024-09-21T10:22:28.835196+0000","last_peered":"2024-09-21T10:22:28.835196+0000","last_clean":"2024-09-21T10:20:12.067444+0000","last_became_active":"2024-09-21T10:22:01.391808+0000","last_became_peered":"2024-09-21T10:22:01.391808+0000","last_unstale":"2024-09-21T10:22:28.835196+0000","last_undegraded":"2024-09-21T10:22:01.123056+0000","last_fullsized":"2024-09-21T10:22:28.835196+0000","mapping_epoch":346,"log_start":"314'1900","ondisk_log_start":"314'1900","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6823,"log_dups_size":0,"ondisk_log_size":6823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251199488,"num_objects":3833,"num_object_clones":0,"num_object_copies":11499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2680,"num_objects_misplaced":4986,"num_objects_unfound":0,"num_objects_dirty":3833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8723,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1033,"num_bytes_recovered":67567616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[3,5,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,5","objects":2566},{"shards":"3,4,6","objects":1267}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":607,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067893+0000","last_change":"2024-09-21T10:22:04.032207+0000","last_active":"2024-09-21T10:22:17.067893+0000","last_peered":"2024-09-21T10:22:17.067893+0000","last_clean":"2024-09-21T10:22:17.067893+0000","last_became_active":"2024-09-21T10:22:04.031881+0000","last_became_peered":"2024-09-21T10:22:04.031881+0000","last_unstale":"2024-09-21T10:22:17.067893+0000","last_undegraded":"2024-09-21T10:22:17.067893+0000","last_fullsized":"2024-09-21T10:22:17.067893+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:08.332410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:08.332410+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":614,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054434+0000","last_change":"2024-09-21T10:21:34.876196+0000","last_active":"2024-09-21T10:22:17.054434+0000","last_peered":"2024-09-21T10:22:17.054434+0000","last_clean":"2024-09-21T10:22:17.054434+0000","last_became_active":"2024-09-21T10:21:34.868031+0000","last_became_peered":"2024-09-21T10:21:34.868031+0000","last_unstale":"2024-09-21T10:22:17.054434+0000","last_undegraded":"2024-09-21T10:22:17.054434+0000","last_fullsized":"2024-09-21T10:22:17.054434+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:58.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":962,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.619694+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:22:29.619694+0000","last_peered":"2024-09-21T10:22:29.619694+0000","last_clean":"2024-09-21T10:22:29.619694+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:22:29.619694+0000","last_undegraded":"2024-09-21T10:22:29.619694+0000","last_fullsized":"2024-09-21T10:22:29.619694+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":610,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052378+0000","last_change":"2024-09-21T10:22:01.329400+0000","last_active":"2024-09-21T10:22:17.052378+0000","last_peered":"2024-09-21T10:22:17.052378+0000","last_clean":"2024-09-21T10:22:17.052378+0000","last_became_active":"2024-09-21T10:22:01.328887+0000","last_became_peered":"2024-09-21T10:22:01.328887+0000","last_unstale":"2024-09-21T10:22:17.052378+0000","last_undegraded":"2024-09-21T10:22:17.052378+0000","last_fullsized":"2024-09-21T10:22:17.052378+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:09.626637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"353'8600","reported_seq":10646,"reported_epoch":353,"state":"active+recovering+degraded+remapped","last_fresh":"2024-09-21T10:22:29.694837+0000","last_change":"2024-09-21T10:21:39.570426+0000","last_active":"2024-09-21T10:22:29.694837+0000","last_peered":"2024-09-21T10:22:29.694837+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:21:39.424206+0000","last_became_peered":"2024-09-21T10:21:39.424206+0000","last_unstale":"2024-09-21T10:22:29.694837+0000","last_undegraded":"2024-09-21T10:21:39.368782+0000","last_fullsized":"2024-09-21T10:22:29.694837+0000","mapping_epoch":339,"log_start":"313'300","ondisk_log_start":"313'300","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":8300,"log_dups_size":0,"ondisk_log_size":8300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:54.426685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247005208,"num_objects":3770,"num_object_clones":0,"num_object_copies":11310,"num_objects_missing_on_primary":2677,"num_objects_missing":2677,"num_objects_degraded":1230,"num_objects_misplaced":1243,"num_objects_unfound":0,"num_objects_dirty":3770,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8600,"num_write_kb":395777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":968,"num_bytes_recovered":63176704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,5,6","objects":2527},{"shards":"2,6","objects":1230}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":582,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:17.068442+0000","last_change":"2024-09-21T10:22:02.009815+0000","last_active":"2024-09-21T10:22:17.068442+0000","last_peered":"2024-09-21T10:22:17.068442+0000","last_clean":"2024-09-21T10:12:59.519603+0000","last_became_active":"2024-09-21T10:22:02.009463+0000","last_became_peered":"2024-09-21T10:22:02.009463+0000","last_unstale":"2024-09-21T10:22:17.068442+0000","last_undegraded":"2024-09-21T10:22:02.002510+0000","last_fullsized":"2024-09-21T10:22:02.001835+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:02.435487+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:02.435487+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1},{"shards":"6,7","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":19,"reported_epoch":352,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:17.066729+0000","last_change":"2024-09-21T10:22:01.318966+0000","last_active":"2024-09-21T10:22:17.066729+0000","last_peered":"2024-09-21T10:22:17.066729+0000","last_clean":"2024-09-21T10:21:59.002104+0000","last_became_active":"2024-09-21T10:22:01.317312+0000","last_became_peered":"2024-09-21T10:22:01.317312+0000","last_unstale":"2024-09-21T10:22:17.066729+0000","last_undegraded":"2024-09-21T10:22:00.997012+0000","last_fullsized":"2024-09-21T10:22:17.066729+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":638,"reported_epoch":352,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:17.055233+0000","last_change":"2024-09-21T10:21:34.866048+0000","last_active":"2024-09-21T10:22:17.055233+0000","last_peered":"2024-09-21T10:22:17.055233+0000","last_clean":"2024-09-21T10:21:17.726080+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:22:17.055233+0000","last_undegraded":"2024-09-21T10:21:34.856315+0000","last_fullsized":"2024-09-21T10:22:17.055233+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":325,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053317+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:22:17.053317+0000","last_peered":"2024-09-21T10:22:17.053317+0000","last_clean":"2024-09-21T10:22:17.053317+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:22:17.053317+0000","last_undegraded":"2024-09-21T10:22:17.053317+0000","last_fullsized":"2024-09-21T10:22:17.053317+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":349,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053876+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:22:17.053876+0000","last_peered":"2024-09-21T10:22:17.053876+0000","last_clean":"2024-09-21T10:22:17.053876+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:22:17.053876+0000","last_undegraded":"2024-09-21T10:22:17.053876+0000","last_fullsized":"2024-09-21T10:22:17.053876+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":59,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053657+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:22:17.053657+0000","last_peered":"2024-09-21T10:22:17.053657+0000","last_clean":"2024-09-21T10:22:17.053657+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:22:17.053657+0000","last_undegraded":"2024-09-21T10:22:17.053657+0000","last_fullsized":"2024-09-21T10:22:17.053657+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.555443+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:22:29.555443+0000","last_peered":"2024-09-21T10:22:29.555443+0000","last_clean":"2024-09-21T10:22:29.555443+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:22:29.555443+0000","last_undegraded":"2024-09-21T10:22:29.555443+0000","last_fullsized":"2024-09-21T10:22:29.555443+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":31,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067765+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:22:17.067765+0000","last_peered":"2024-09-21T10:22:17.067765+0000","last_clean":"2024-09-21T10:22:17.067765+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:22:17.067765+0000","last_undegraded":"2024-09-21T10:22:17.067765+0000","last_fullsized":"2024-09-21T10:22:17.067765+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.051416+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:22:17.051416+0000","last_peered":"2024-09-21T10:22:17.051416+0000","last_clean":"2024-09-21T10:22:17.051416+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:22:17.051416+0000","last_undegraded":"2024-09-21T10:22:17.051416+0000","last_fullsized":"2024-09-21T10:22:17.051416+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":533,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.051711+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:22:17.051711+0000","last_peered":"2024-09-21T10:22:17.051711+0000","last_clean":"2024-09-21T10:22:17.051711+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:22:17.051711+0000","last_undegraded":"2024-09-21T10:22:17.051711+0000","last_fullsized":"2024-09-21T10:22:17.051711+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":189,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067223+0000","last_change":"2024-09-21T10:20:13.089473+0000","last_active":"2024-09-21T10:22:17.067223+0000","last_peered":"2024-09-21T10:22:17.067223+0000","last_clean":"2024-09-21T10:22:17.067223+0000","last_became_active":"2024-09-21T10:20:13.089111+0000","last_became_peered":"2024-09-21T10:20:13.089111+0000","last_unstale":"2024-09-21T10:22:17.067223+0000","last_undegraded":"2024-09-21T10:22:17.067223+0000","last_fullsized":"2024-09-21T10:22:17.067223+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:06.603284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":593,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052284+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:22:17.052284+0000","last_peered":"2024-09-21T10:22:17.052284+0000","last_clean":"2024-09-21T10:22:17.052284+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:22:17.052284+0000","last_undegraded":"2024-09-21T10:22:17.052284+0000","last_fullsized":"2024-09-21T10:22:17.052284+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":366,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.056593+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:22:17.056593+0000","last_peered":"2024-09-21T10:22:17.056593+0000","last_clean":"2024-09-21T10:22:17.056593+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:22:17.056593+0000","last_undegraded":"2024-09-21T10:22:17.056593+0000","last_fullsized":"2024-09-21T10:22:17.056593+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":610,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.052455+0000","last_change":"2024-09-21T10:22:01.329690+0000","last_active":"2024-09-21T10:22:17.052455+0000","last_peered":"2024-09-21T10:22:17.052455+0000","last_clean":"2024-09-21T10:22:17.052455+0000","last_became_active":"2024-09-21T10:22:01.329033+0000","last_became_peered":"2024-09-21T10:22:01.329033+0000","last_unstale":"2024-09-21T10:22:17.052455+0000","last_undegraded":"2024-09-21T10:22:17.052455+0000","last_fullsized":"2024-09-21T10:22:17.052455+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:24.559524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":192,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067175+0000","last_change":"2024-09-21T10:21:34.870704+0000","last_active":"2024-09-21T10:22:17.067175+0000","last_peered":"2024-09-21T10:22:17.067175+0000","last_clean":"2024-09-21T10:22:17.067175+0000","last_became_active":"2024-09-21T10:21:34.869835+0000","last_became_peered":"2024-09-21T10:21:34.869835+0000","last_unstale":"2024-09-21T10:22:17.067175+0000","last_undegraded":"2024-09-21T10:22:17.067175+0000","last_fullsized":"2024-09-21T10:22:17.067175+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:34.959915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":310,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-21T10:22:29.619291+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:22:29.619291+0000","last_peered":"2024-09-21T10:22:29.619291+0000","last_clean":"2024-09-21T10:22:29.619291+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:22:29.619291+0000","last_undegraded":"2024-09-21T10:22:29.619291+0000","last_fullsized":"2024-09-21T10:22:29.619291+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":600,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054173+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:22:17.054173+0000","last_peered":"2024-09-21T10:22:17.054173+0000","last_clean":"2024-09-21T10:22:17.054173+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:22:17.054173+0000","last_undegraded":"2024-09-21T10:22:17.054173+0000","last_fullsized":"2024-09-21T10:22:17.054173+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":356,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.055497+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:22:17.055497+0000","last_peered":"2024-09-21T10:22:17.055497+0000","last_clean":"2024-09-21T10:22:17.055497+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:22:17.055497+0000","last_undegraded":"2024-09-21T10:22:17.055497+0000","last_fullsized":"2024-09-21T10:22:17.055497+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":599,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.059984+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:22:17.059984+0000","last_peered":"2024-09-21T10:22:17.059984+0000","last_clean":"2024-09-21T10:22:17.059984+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:22:17.059984+0000","last_undegraded":"2024-09-21T10:22:17.059984+0000","last_fullsized":"2024-09-21T10:22:17.059984+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":22,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.066524+0000","last_change":"2024-09-21T10:22:01.318549+0000","last_active":"2024-09-21T10:22:17.066524+0000","last_peered":"2024-09-21T10:22:17.066524+0000","last_clean":"2024-09-21T10:22:17.066524+0000","last_became_active":"2024-09-21T10:22:01.318071+0000","last_became_peered":"2024-09-21T10:22:01.318071+0000","last_unstale":"2024-09-21T10:22:17.066524+0000","last_undegraded":"2024-09-21T10:22:17.066524+0000","last_fullsized":"2024-09-21T10:22:17.066524+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:49.061492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":49,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:19.978676+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:22:19.978676+0000","last_peered":"2024-09-21T10:22:19.978676+0000","last_clean":"2024-09-21T10:22:19.978676+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:22:19.978676+0000","last_undegraded":"2024-09-21T10:22:19.978676+0000","last_fullsized":"2024-09-21T10:22:19.978676+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":291,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.051520+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:22:17.051520+0000","last_peered":"2024-09-21T10:22:17.051520+0000","last_clean":"2024-09-21T10:22:17.051520+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:22:17.051520+0000","last_undegraded":"2024-09-21T10:22:17.051520+0000","last_fullsized":"2024-09-21T10:22:17.051520+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":49,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:19.978627+0000","last_change":"2024-09-21T10:19:50.588363+0000","last_active":"2024-09-21T10:22:19.978627+0000","last_peered":"2024-09-21T10:22:19.978627+0000","last_clean":"2024-09-21T10:22:19.978627+0000","last_became_active":"2024-09-21T10:19:50.588107+0000","last_became_peered":"2024-09-21T10:19:50.588107+0000","last_unstale":"2024-09-21T10:22:19.978627+0000","last_undegraded":"2024-09-21T10:22:19.978627+0000","last_fullsized":"2024-09-21T10:22:19.978627+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:05.698018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":269,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.067180+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:22:17.067180+0000","last_peered":"2024-09-21T10:22:17.067180+0000","last_clean":"2024-09-21T10:22:17.067180+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:22:17.067180+0000","last_undegraded":"2024-09-21T10:22:17.067180+0000","last_fullsized":"2024-09-21T10:22:17.067180+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":613,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053821+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:22:17.053821+0000","last_peered":"2024-09-21T10:22:17.053821+0000","last_clean":"2024-09-21T10:22:17.053821+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:22:17.053821+0000","last_undegraded":"2024-09-21T10:22:17.053821+0000","last_fullsized":"2024-09-21T10:22:17.053821+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":575,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.054029+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:22:17.054029+0000","last_peered":"2024-09-21T10:22:17.054029+0000","last_clean":"2024-09-21T10:22:17.054029+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:22:17.054029+0000","last_undegraded":"2024-09-21T10:22:17.054029+0000","last_fullsized":"2024-09-21T10:22:17.054029+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":49,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-21T10:22:17.053046+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:22:17.053046+0000","last_peered":"2024-09-21T10:22:17.053046+0000","last_clean":"2024-09-21T10:22:17.053046+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:22:17.053046+0000","last_undegraded":"2024-09-21T10:22:17.053046+0000","last_fullsized":"2024-09-21T10:22:17.053046+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":19,"reported_epoch":352,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:17.066906+0000","last_change":"2024-09-21T10:22:01.317167+0000","last_active":"2024-09-21T10:22:17.066906+0000","last_peered":"2024-09-21T10:22:17.066906+0000","last_clean":"2024-09-21T10:14:43.015500+0000","last_became_active":"2024-09-21T10:22:01.316569+0000","last_became_peered":"2024-09-21T10:22:01.316569+0000","last_unstale":"2024-09-21T10:22:17.066906+0000","last_undegraded":"2024-09-21T10:22:00.997069+0000","last_fullsized":"2024-09-21T10:22:17.066906+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4043833368,"num_objects":61705,"num_object_clones":0,"num_object_copies":185115,"num_objects_missing_on_primary":2677,"num_objects_missing":2677,"num_objects_degraded":19133,"num_objects_misplaced":10023,"num_objects_unfound":0,"num_objects_dirty":61705,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137981,"num_write_kb":6389889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17969,"num_bytes_recovered":1086521344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12365737984,"data_stored":12365725768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105952,"ondisk_log_size":105952,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":3,"num_objects_missing":3,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24225,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":391,"num_read_kb":833,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":3107188,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1640,"ondisk_log_size":1640,"up":33,"acting":31,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234394,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1344616,"kb_used_data":754016,"kb_used_omap":1,"kb_used_meta":590590,"kb_avail":92400536,"statfs":{"total":95995035648,"available":94618148864,"internally_reserved":0,"allocated":772112384,"data_stored":770317670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":604764619},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496032,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2567920,"kb_used_data":1955176,"kb_used_omap":1,"kb_used_meta":612734,"kb_avail":91177232,"statfs":{"total":95995035648,"available":93365485568,"internally_reserved":0,"allocated":2002100224,"data_stored":2000298672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":627440071},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403932,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969056,"kb_used_data":1746776,"kb_used_omap":7,"kb_used_meta":222264,"kb_avail":91776096,"statfs":{"total":95995035648,"available":93978722304,"internally_reserved":0,"allocated":1788698624,"data_stored":1786908156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":227599256},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019305,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2071528,"kb_used_data":1490784,"kb_used_omap":1,"kb_used_meta":580734,"kb_avail":91673624,"statfs":{"total":95995035648,"available":93873790976,"internally_reserved":0,"allocated":1526562816,"data_stored":1524768363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":594672070},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435489,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2745364,"kb_used_data":1991756,"kb_used_omap":5,"kb_used_meta":753594,"kb_avail":90999788,"statfs":{"total":95995035648,"available":93183782912,"internally_reserved":0,"allocated":2039558144,"data_stored":2037779371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":771680296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599016,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2167964,"kb_used_data":1486548,"kb_used_omap":7,"kb_used_meta":681400,"kb_avail":91577188,"statfs":{"total":95995035648,"available":93775040512,"internally_reserved":0,"allocated":1522225152,"data_stored":1520438973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":697754061},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762543,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2324460,"kb_used_data":1733280,"kb_used_omap":7,"kb_used_meta":591160,"kb_avail":91420692,"statfs":{"total":95995035648,"available":93614788608,"internally_reserved":0,"allocated":1774878720,"data_stored":1773088338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":605348093},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926069,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1450964,"kb_used_data":1013900,"kb_used_omap":7,"kb_used_meta":437048,"kb_avail":92294188,"statfs":{"total":95995035648,"available":94509248512,"internally_reserved":0,"allocated":1038233600,"data_stored":1036454759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":447537342},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1024065536,"data_stored":1024065536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1754595328,"data_stored":1754595328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1508380672,"data_stored":1508376600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2023620608,"data_stored":2023620608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1518206976,"data_stored":1518206976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1782583296,"data_stored":1782579224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1993281536,"data_stored":1993277464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":761004032,"data_stored":761004032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:22:33.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:22:33.216 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:33.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:33.345 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:33.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:33.446 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:33.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:33.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:33.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:33.601 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:33.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:33.688 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:33.704 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:33.729 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:33.730 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:33.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:33.822 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:33 smithi032 ceph-mon[105748]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:22:33.822 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:33 smithi032 ceph-mon[105748]: pgmap v1053: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 19158/184500 objects degraded (10.384%); 10023/184500 objects misplaced (5.433%); 5.9 MiB/s, 211 objects/s recovering 2024-09-21T10:22:33.822 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:33.822 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:33.822 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:33 smithi032 ceph-mon[101299]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:22:33.822 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:33 smithi032 ceph-mon[101299]: pgmap v1053: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 19158/184500 objects degraded (10.384%); 10023/184500 objects misplaced (5.433%); 5.9 MiB/s, 211 objects/s recovering 2024-09-21T10:22:33.822 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:33.823 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:33.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:33.855 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:33.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:33.957 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:33.979 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:33.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:33 smithi045 ceph-mon[95976]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:22:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:33 smithi045 ceph-mon[95976]: pgmap v1053: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 19158/184500 objects degraded (10.384%); 10023/184500 objects misplaced (5.433%); 5.9 MiB/s, 211 objects/s recovering 2024-09-21T10:22:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:34.081 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:34.105 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:34.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:34.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:34.331 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:34.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:34.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:34.456 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:34.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:34.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:34.611 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:34.645 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:34.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:34.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:34 smithi032 ceph-mon[101299]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T10:22:34.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:34 smithi032 ceph-mon[105748]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T10:22:34.747 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:34.789 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:34.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:34.931 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:34.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:35.033 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:35.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:34 smithi045 ceph-mon[95976]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T10:22:35.068 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:35.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:35.169 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:35.200 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:35.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:35.302 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:35.327 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:35.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:35.339 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:22:35.340 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:22:35.340 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:22:35.340 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:22:35.340 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:22:35.340 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:22:35.340 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:22:35.340 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:22:35.340 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:22:35.341 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:22:35.341 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:22:35.341 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:22:35.341 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:22:35.341 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:22:35.341 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:22:35.341 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:22:35.428 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:35.454 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:35.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:35.556 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:35.584 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:35.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:35.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:35.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:35 smithi032 ceph-mon[105748]: pgmap v1055: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 19049/182739 objects degraded (10.424%); 9912/182739 objects misplaced (5.424%); 3.6 MiB/s, 157 objects/s recovering 2024-09-21T10:22:35.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:35 smithi032 ceph-mon[105748]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T10:22:35.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:35 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1909576426' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:35.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:35 smithi032 ceph-mon[101299]: pgmap v1055: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 19049/182739 objects degraded (10.424%); 9912/182739 objects misplaced (5.424%); 3.6 MiB/s, 157 objects/s recovering 2024-09-21T10:22:35.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:35 smithi032 ceph-mon[101299]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T10:22:35.709 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:35 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1909576426' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:35.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:35.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:35.812 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:35.839 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:35.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:35.941 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:35.971 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:35.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:35 smithi045 ceph-mon[95976]: pgmap v1055: 99 pgs: 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 19049/182739 objects degraded (10.424%); 9912/182739 objects misplaced (5.424%); 3.6 MiB/s, 157 objects/s recovering 2024-09-21T10:22:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:35 smithi045 ceph-mon[95976]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T10:22:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:35 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1909576426' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:36.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:36.111 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:36.212 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:36.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:36.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:36.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:36.370 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:36.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:36.471 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:36.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:36.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:36.599 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:36.630 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:36.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:36.731 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:36.756 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:36.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:36.858 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:36.880 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:36.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:36.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:37.008 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:37.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:37.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:37.136 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:37.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:37.238 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:37.270 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:37.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:37.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.403 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:37.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:37.504 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:37.531 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:37.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:37.633 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:37.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:37.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:37.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:37.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:37.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:37.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:37.910 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:37.914 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:37 smithi032 ceph-mon[101299]: pgmap v1057: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 430 op/s; 18217/179364 objects degraded (10.156%); 9665/179364 objects misplaced (5.388%); 5.2 MiB/s, 282 objects/s recovering 2024-09-21T10:22:37.914 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:37 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 18217/179364 objects degraded (10.156%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:37.914 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:37 smithi032 ceph-mon[105748]: pgmap v1057: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 430 op/s; 18217/179364 objects degraded (10.156%); 9665/179364 objects misplaced (5.388%); 5.2 MiB/s, 282 objects/s recovering 2024-09-21T10:22:37.914 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:37 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 18217/179364 objects degraded (10.156%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:38.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:37 smithi045 ceph-mon[95976]: pgmap v1057: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 430 op/s; 18217/179364 objects degraded (10.156%); 9665/179364 objects misplaced (5.388%); 5.2 MiB/s, 282 objects/s recovering 2024-09-21T10:22:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:37 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 18217/179364 objects degraded (10.156%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:38.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:38.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:38.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:38.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:22:38.169 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:38.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:38.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:38.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.301 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:38.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:38.402 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:38.431 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:38.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:38.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:38.556 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:38.658 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:38.673 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:22:38.673 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:22:38.684 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:38.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:38.786 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:38.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:38.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:38.913 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:38.939 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:38.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:39.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:39.066 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:39.168 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:39.193 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:39.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:39.295 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:39.319 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:39.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:39.421 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:39.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:39.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:39.548 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:39.572 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:39.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:39.581 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:22:39.674 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:39.691 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:22:39.701 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:39.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:39.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:39.832 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:39.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:39.934 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:39 smithi032 ceph-mon[105748]: pgmap v1058: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 277 op/s; 18217/178527 objects degraded (10.204%); 9665/178527 objects misplaced (5.414%); 2.8 MiB/s, 119 objects/s recovering 2024-09-21T10:22:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1112988693' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:39 smithi032 ceph-mon[101299]: pgmap v1058: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 277 op/s; 18217/178527 objects degraded (10.204%); 9665/178527 objects misplaced (5.414%); 2.8 MiB/s, 119 objects/s recovering 2024-09-21T10:22:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1112988693' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:22:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:22:39] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-09-21T10:22:39.965 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:39.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:39 smithi045 ceph-mon[95976]: pgmap v1058: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 277 op/s; 18217/178527 objects degraded (10.204%); 9665/178527 objects misplaced (5.414%); 2.8 MiB/s, 119 objects/s recovering 2024-09-21T10:22:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:39 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1112988693' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:40.066 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:40.090 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:40.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:40.191 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:40.217 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:40.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:40.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:40.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:40.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:40.447 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:40.475 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:40.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:40.577 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:40.600 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:40.628 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:40.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:40.726 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:40.828 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:40.854 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:40.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:40.955 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:40.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:40 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:40.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:40 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1112988693' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:40.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:40 smithi032 ceph-mon[101299]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T10:22:40.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:40 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:40.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:40 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1112988693' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:40.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:40 smithi032 ceph-mon[105748]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T10:22:40.978 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:41.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:40 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:41.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:40 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1112988693' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:41.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:40 smithi045 ceph-mon[95976]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T10:22:41.079 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:41.107 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:41.209 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:41.241 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:41.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:41.371 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:41.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:41.472 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:41.501 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:41.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:41.512 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:41.602 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.637 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:41.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:41.738 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:41.763 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:41.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:41.865 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:41.888 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:41.894 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:41 smithi032 ceph-mon[101299]: pgmap v1060: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 17815/172731 objects degraded (10.314%); 9271/172731 objects misplaced (5.367%); 6.7 MiB/s, 193 objects/s recovering 2024-09-21T10:22:41.894 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:41 smithi032 ceph-mon[105748]: pgmap v1060: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 17815/172731 objects degraded (10.314%); 9271/172731 objects misplaced (5.367%); 6.7 MiB/s, 193 objects/s recovering 2024-09-21T10:22:41.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:42.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:42.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:42.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:41 smithi045 ceph-mon[95976]: pgmap v1060: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 17815/172731 objects degraded (10.314%); 9271/172731 objects misplaced (5.367%); 6.7 MiB/s, 193 objects/s recovering 2024-09-21T10:22:42.114 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:42.142 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:42.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:42.244 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:42.282 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:42.383 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:42.407 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:42.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:42.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:42.532 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:42.606 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:22:42.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:42.680 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:42.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:42.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:42 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 17815/172731 objects degraded (10.314%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:42.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:42 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 17815/172731 objects degraded (10.314%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:42.781 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:42.810 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:42.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:42.912 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:42.940 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:42 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 17815/172731 objects degraded (10.314%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:43.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:43.066 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:43.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:43.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:43.194 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:43.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:43.296 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:43.320 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:43.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:43.422 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:43.451 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:43.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:43.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:43.586 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:43.690 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:43.696 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:43.723 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:43.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:43.825 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:43.852 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:43.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:43.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:43 smithi032 ceph-mon[105748]: pgmap v1061: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 17815/172002 objects degraded (10.357%); 9271/172002 objects misplaced (5.390%); 5.0 MiB/s, 154 objects/s recovering 2024-09-21T10:22:43.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:43 smithi032 ceph-mon[101299]: pgmap v1061: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 17815/172002 objects degraded (10.357%); 9271/172002 objects misplaced (5.390%); 5.0 MiB/s, 154 objects/s recovering 2024-09-21T10:22:43.986 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:43 smithi045 ceph-mon[95976]: pgmap v1061: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 17815/172002 objects degraded (10.357%); 9271/172002 objects misplaced (5.390%); 5.0 MiB/s, 154 objects/s recovering 2024-09-21T10:22:44.088 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:44.119 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:44.220 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:44.252 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:44.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:44.354 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:44.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.382 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:44.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:44.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:44.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:44.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:44.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.653 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:44.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:44.754 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:44.783 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:44.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:44.885 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:44.886 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:22:44.886 INFO:teuthology.orchestra.run.smithi032.stdout:{"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 17674/169938 objects degraded (10.400%), 14 pgs degraded, 6 pgs undersized","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1455,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":356,"num_osds":8,"num_up_osds":8,"osd_up_since":1726914012,"num_in_osds":8,"osd_in_since":1726913989,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":56646,"data_bytes":3715241981,"bytes_used":16201093120,"bytes_avail":751759192064,"bytes_total":767960285184,"degraded_objects":17674,"degraded_total":169938,"degraded_ratio":0.10400263625557554,"misplaced_objects":9130,"misplaced_total":169938,"misplaced_ratio":0.053725476350198316,"recovering_objects_per_sec":138,"recovering_bytes_per_sec":5147636,"recovering_keys_per_sec":0,"num_objects_recovered":1378,"num_bytes_recovered":51249152,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":428},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi032.fdolcm","status":"up:active","gid":14655}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.32:8443/","prometheus":"http://172.21.15.32:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-21T10:21:12.493782+0000","services":{}},"progress_events":{}} 2024-09-21T10:22:44.914 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:44.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:45.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:45.038 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:45.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:45.169 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:45.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:45.271 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:45.301 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:45.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:45.425 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:45.527 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:45.560 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:45.662 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:45.690 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:45.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:45.818 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:45.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:45.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:45.943 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:45.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:45.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:45 smithi032 ceph-mon[101299]: pgmap v1062: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 17674/169938 objects degraded (10.400%); 9130/169938 objects misplaced (5.373%); 4.9 MiB/s, 138 objects/s recovering 2024-09-21T10:22:45.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:45 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4015770431' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:22:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:45 smithi032 ceph-mon[105748]: pgmap v1062: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 17674/169938 objects degraded (10.400%); 9130/169938 objects misplaced (5.373%); 4.9 MiB/s, 138 objects/s recovering 2024-09-21T10:22:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:45 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4015770431' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:22:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:45 smithi045 ceph-mon[95976]: pgmap v1062: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 17674/169938 objects degraded (10.400%); 9130/169938 objects misplaced (5.373%); 4.9 MiB/s, 138 objects/s recovering 2024-09-21T10:22:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:45 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4015770431' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:22:46.045 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:46.073 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:46.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:46.175 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:46.197 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:46.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:46.299 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:46.323 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:46.345 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:46.425 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:46.452 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:46.554 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:46.580 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:46.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:46.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:46.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:46.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:46.813 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:46.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:46.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:46.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:46.973 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:46.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:47.074 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:47.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:47.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:47.234 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:47.336 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:47.364 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:47.466 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:47.492 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:47.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:47.594 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:47.618 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:47 smithi032 ceph-mon[105748]: pgmap v1063: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 17397/166035 objects degraded (10.478%); 8854/166035 objects misplaced (5.333%); 5.1 MiB/s, 80 objects/s recovering 2024-09-21T10:22:47.618 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:47 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 17397/166035 objects degraded (10.478%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:47.618 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:47 smithi032 ceph-mon[101299]: pgmap v1063: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 17397/166035 objects degraded (10.478%); 8854/166035 objects misplaced (5.333%); 5.1 MiB/s, 80 objects/s recovering 2024-09-21T10:22:47.618 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:47 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 17397/166035 objects degraded (10.478%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:47.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:47.724 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:47.758 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:47.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:47.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:47.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:47.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:47.991 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:48.021 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:47 smithi045 ceph-mon[95976]: pgmap v1063: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 17397/166035 objects degraded (10.478%); 8854/166035 objects misplaced (5.333%); 5.1 MiB/s, 80 objects/s recovering 2024-09-21T10:22:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:47 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 17397/166035 objects degraded (10.478%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:48.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:48.151 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:48.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:48.252 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:48.277 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:48.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:48.349 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:22:48.349 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:22:48.368 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:22:48.368 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:22:48.379 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:48.387 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:22:48.388 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:22:48.404 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:48.409 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:22:48.409 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:22:48.427 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:22:48.427 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:22:48.447 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:22:48.448 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:22:48.467 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:22:48.467 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:22:48.487 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:22:48.487 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:22:48.506 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:48.539 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:48.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:48.641 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:48.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:48.767 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:48.793 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:48.895 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:48.923 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:48.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:48 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2979807156' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:48.948 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:48.948 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:48 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2979807156' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:48.948 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:48.974 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:22:48.975 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:22:49.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:49.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:48 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2979807156' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:49.047 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:49.149 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:49.176 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:49.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:49.278 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:49.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:49.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:49.412 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:49.444 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:49.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:49.546 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:49.577 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:49.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:49.610 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:22:49.679 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:49.703 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:49.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:49.805 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:49.830 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:49.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:49.932 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:49.955 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:49.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:49.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:49 smithi032 ceph-mon[101299]: pgmap v1064: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 17397/165183 objects degraded (10.532%); 8854/165183 objects misplaced (5.360%); 5.1 MiB/s, 80 objects/s recovering 2024-09-21T10:22:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:49 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2588060102' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:49 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:49 smithi032 ceph-mon[105748]: pgmap v1064: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 17397/165183 objects degraded (10.532%); 8854/165183 objects misplaced (5.360%); 5.1 MiB/s, 80 objects/s recovering 2024-09-21T10:22:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:49 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2588060102' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:49 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:49.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:22:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:22:49] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-21T10:22:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:49 smithi045 ceph-mon[95976]: pgmap v1064: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 17397/165183 objects degraded (10.532%); 8854/165183 objects misplaced (5.360%); 5.1 MiB/s, 80 objects/s recovering 2024-09-21T10:22:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:49 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2588060102' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:49 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:50.056 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:50.079 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:50.180 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:50.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:50.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:50.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:50.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:50.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:50.439 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:50.469 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:50.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:50.570 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:50.599 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:50.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:50.733 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:50.835 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:50.868 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:50.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:50.907 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:50 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:50.907 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:50 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:50.907 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:50 smithi032 ceph-mon[101299]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T10:22:50.907 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:50 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:50.907 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:50 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:50.907 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:50 smithi032 ceph-mon[105748]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T10:22:50.970 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:51.001 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:51.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:50 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:50 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:50 smithi045 ceph-mon[95976]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T10:22:51.103 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:51.150 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:51.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:51.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:51.282 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:51.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:51.414 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:51.516 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:51.541 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:51.643 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:51.677 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:51.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:51 smithi032 ceph-mon[105748]: pgmap v1066: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 16971/159219 objects degraded (10.659%); 8428/159219 objects misplaced (5.293%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:22:51.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:22:51.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:51 smithi032 ceph-mon[101299]: pgmap v1066: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 16971/159219 objects degraded (10.659%); 8428/159219 objects misplaced (5.293%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:22:51.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:22:51.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:51 smithi045 ceph-mon[95976]: pgmap v1066: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 16971/159219 objects degraded (10.659%); 8428/159219 objects misplaced (5.293%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:22:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:22:51.810 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:51.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:51.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:51.942 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:51.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:52.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:52.068 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:52.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:52.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:22:52.170 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:52.205 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:52.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:52.291 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:52.307 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:52.319 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:22:52.319 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:22:52.319 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:22:52.319 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:22:52.319 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:22:52.343 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:52.444 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:52.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:52.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:52.573 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:52.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:52.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:52.709 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:52.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:52.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:52.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:52 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16971/159219 objects degraded (10.659%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:52.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:52.881 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:52.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:52.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:52 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16971/159219 objects degraded (10.659%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:52.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:52 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16971/159219 objects degraded (10.659%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:52.982 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:53.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:53.132 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:53.176 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:53.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:53.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:53.471 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:53.556 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:22:53.571 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:53.574 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:53.619 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:53.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:53.755 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:53.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:53.773 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:22:53.831 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:53 smithi032 ceph-mon[101299]: pgmap v1067: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 16971/158358 objects degraded (10.717%); 8428/158358 objects misplaced (5.322%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:22:53.831 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:53 smithi032 ceph-mon[105748]: pgmap v1067: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 16971/158358 objects degraded (10.717%); 8428/158358 objects misplaced (5.322%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:22:53.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:53.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:53.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:53.991 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:54.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.019 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:54.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:54.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:53 smithi045 ceph-mon[95976]: pgmap v1067: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 16971/158358 objects degraded (10.717%); 8428/158358 objects misplaced (5.322%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:22:54.120 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:54.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:54.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:54.251 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:54.275 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:54.377 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.410 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:54.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:54.511 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:54.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:54.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:54.642 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:54.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:54.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:54.776 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:54.815 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:54.916 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:54.949 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:54.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:55.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:55.077 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:55.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:55.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:55.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:55.352 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:55.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:55.454 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:55.484 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:55.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:55.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:55.616 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:55.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:55.718 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:55.745 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:55.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:55.847 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:55.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:55.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:55.908 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:55 smithi032 ceph-mon[101299]: pgmap v1068: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 16807/155967 objects degraded (10.776%); 8264/155967 objects misplaced (5.299%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:22:55.908 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:55 smithi032 ceph-mon[105748]: pgmap v1068: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 16807/155967 objects degraded (10.776%); 8264/155967 objects misplaced (5.299%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:22:55.982 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:56.017 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:56.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:56.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:55 smithi045 ceph-mon[95976]: pgmap v1068: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 16807/155967 objects degraded (10.776%); 8264/155967 objects misplaced (5.299%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:22:56.118 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:56.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:56.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:56.255 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:56.280 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:56.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:56.382 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:56.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:56.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:56.544 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:56.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:56.646 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:56.677 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:56.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:56.812 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:56.913 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:56.944 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:56.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:56 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:56 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:56 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16807/155967 objects degraded (10.776%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:56 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:56 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:56 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16807/155967 objects degraded (10.776%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:57.047 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:57.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:57.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:57.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:57.205 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:57.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:56 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:56 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:22:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:56 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16807/155967 objects degraded (10.776%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:22:57.307 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.336 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:57.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:57.437 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:57.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:57.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:57.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:57.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:57.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:57.741 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:57.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:57.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:57.871 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:57.972 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:58.003 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:58.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:58.104 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:58.105 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:57 smithi032 ceph-mon[101299]: pgmap v1069: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 16486/151596 objects degraded (10.875%); 7941/151596 objects misplaced (5.238%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:22:58.105 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:57 smithi032 ceph-mon[105748]: pgmap v1069: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 16486/151596 objects degraded (10.875%); 7941/151596 objects misplaced (5.238%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:22:58.118 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:58.147 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:58.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:58.248 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:58.279 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:57 smithi045 ceph-mon[95976]: pgmap v1069: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 16486/151596 objects degraded (10.875%); 7941/151596 objects misplaced (5.238%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:22:58.382 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.416 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:58.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:58.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:58.546 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:58.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:58.648 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:58.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:58.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:58.776 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:58.801 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:58.903 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:58.933 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:59.035 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:59.071 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:59.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:59.198 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:59.300 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:59.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:59.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:59.435 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:59.468 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:59.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:59.569 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:59.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.600 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:59.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:59.616 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:59.700 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:59.736 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:22:59.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:59.752 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:22:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:22:59] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-21T10:22:59.838 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:59.877 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:22:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:59.979 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:59.980 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:22:59.980 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:23:00.012 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:00.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:00.017 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:22:59 smithi032 ceph-mon[101299]: pgmap v1070: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 16486/150594 objects degraded (10.947%); 7941/150594 objects misplaced (5.273%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:23:00.017 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:22:59 smithi032 ceph-mon[105748]: pgmap v1070: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 16486/150594 objects degraded (10.947%); 7941/150594 objects misplaced (5.273%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:23:00.017 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:23:00.018 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:23:00.045 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:23:00.045 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:23:00.072 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:23:00.072 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:23:00.100 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:23:00.100 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:23:00.114 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:00.131 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:23:00.131 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:23:00.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:00.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:00.157 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:23:00.157 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:23:00.183 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:23:00.183 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:23:00.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:00.280 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:00.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:00.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:22:59 smithi045 ceph-mon[95976]: pgmap v1070: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 16486/150594 objects degraded (10.947%); 7941/150594 objects misplaced (5.273%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:23:00.382 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:00.416 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:00.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:00.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:00.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:00.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:00.686 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:00.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:00.788 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:00.819 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:00.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:00.920 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.958 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:00.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:01.012 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:23:01.014 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:23:01.059 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:01.091 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:01.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:01.193 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:00 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1210916265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:00 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1210916265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:01.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:01.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:01.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:00 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1210916265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:01.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:01.349 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:01.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:01.451 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:01.489 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:01.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:01.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:01.621 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:01.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:01.722 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:01.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.755 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:01.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:01.856 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:01.890 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:01.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:01.992 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:02.024 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:02.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:02.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:02.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:01 smithi032 ceph-mon[105748]: pgmap v1071: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 15897/142617 objects degraded (11.147%); 7350/142617 objects misplaced (5.154%); 7.9 MiB/s, 125 objects/s recovering 2024-09-21T10:23:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:01 smithi032 ceph-mon[105748]: from='client.45728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:23:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:01 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 15897/142617 objects degraded (11.147%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:01 smithi032 ceph-mon[101299]: pgmap v1071: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 15897/142617 objects degraded (11.147%); 7350/142617 objects misplaced (5.154%); 7.9 MiB/s, 125 objects/s recovering 2024-09-21T10:23:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:01 smithi032 ceph-mon[101299]: from='client.45728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:23:02.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:01 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 15897/142617 objects degraded (11.147%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:02.259 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:01 smithi045 ceph-mon[95976]: pgmap v1071: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 15897/142617 objects degraded (11.147%); 7350/142617 objects misplaced (5.154%); 7.9 MiB/s, 125 objects/s recovering 2024-09-21T10:23:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:01 smithi045 ceph-mon[95976]: from='client.45728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:23:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:01 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 15897/142617 objects degraded (11.147%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:02.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:02.387 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:02.415 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:02.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:02.517 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:02.544 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:02.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:02.645 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:02.677 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:02.778 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:02.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:02.913 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:02.942 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:02.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:03.044 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:03.068 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:03.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:03.170 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:03.200 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1071,"stamp":"2024-09-21T10:23:00.530871+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3118405629,"num_objects":47539,"num_object_clones":0,"num_object_copies":142617,"num_objects_missing_on_primary":1801,"num_objects_missing":1801,"num_objects_degraded":15897,"num_objects_misplaced":7350,"num_objects_unfound":0,"num_objects_dirty":47539,"num_whiteouts":0,"num_read":6389,"num_read_kb":13844,"num_write":153298,"num_write_kb":6404930,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21514,"num_bytes_recovered":1281523705,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109619,"ondisk_log_size":109619,"up":297,"acting":289,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14334396,"kb_used_data":9761980,"kb_used_omap":40,"kb_used_meta":4572247,"kb_avail":735626820,"statfs":{"total":767960285184,"available":753281863680,"internally_reserved":0,"allocated":9996267520,"data_stored":9981649861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41824,"internal_metadata":4681981088},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,3,5,3,1,1,2,1,5,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-443744256,"num_objects":-6771,"num_object_clones":0,"num_object_copies":-20313,"num_objects_missing_on_primary":-495,"num_objects_missing":-495,"num_objects_degraded":-1368,"num_objects_misplaced":-1372,"num_objects_unfound":0,"num_objects_dirty":-6771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6771,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1373,"num_bytes_recovered":89980928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.926355"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":591,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.608400+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:22:49.608400+0000","last_peered":"2024-09-21T10:22:49.608400+0000","last_clean":"2024-09-21T10:22:49.608400+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:22:49.608400+0000","last_undegraded":"2024-09-21T10:22:49.608400+0000","last_fullsized":"2024-09-21T10:22:49.608400+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":359,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624458+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:22:39.624458+0000","last_peered":"2024-09-21T10:22:39.624458+0000","last_clean":"2024-09-21T10:22:39.624458+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:22:39.624458+0000","last_undegraded":"2024-09-21T10:22:39.624458+0000","last_fullsized":"2024-09-21T10:22:39.624458+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.611942+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:22:39.611942+0000","last_peered":"2024-09-21T10:22:39.611942+0000","last_clean":"2024-09-21T10:22:39.611942+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:22:39.611942+0000","last_undegraded":"2024-09-21T10:22:39.611942+0000","last_fullsized":"2024-09-21T10:22:39.611942+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":32,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.616302+0000","last_change":"2024-09-21T10:22:01.318638+0000","last_active":"2024-09-21T10:22:49.616302+0000","last_peered":"2024-09-21T10:22:49.616302+0000","last_clean":"2024-09-21T10:22:49.616302+0000","last_became_active":"2024-09-21T10:22:01.317087+0000","last_became_peered":"2024-09-21T10:22:01.317087+0000","last_unstale":"2024-09-21T10:22:49.616302+0000","last_undegraded":"2024-09-21T10:22:49.616302+0000","last_fullsized":"2024-09-21T10:22:49.616302+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:25.334726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":33,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.617915+0000","last_change":"2024-09-21T10:22:01.318978+0000","last_active":"2024-09-21T10:22:49.617915+0000","last_peered":"2024-09-21T10:22:49.617915+0000","last_clean":"2024-09-21T10:22:49.617915+0000","last_became_active":"2024-09-21T10:22:01.318255+0000","last_became_peered":"2024-09-21T10:22:01.318255+0000","last_unstale":"2024-09-21T10:22:49.617915+0000","last_undegraded":"2024-09-21T10:22:49.617915+0000","last_fullsized":"2024-09-21T10:22:49.617915+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:55.084088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":330,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624342+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:22:39.624342+0000","last_peered":"2024-09-21T10:22:39.624342+0000","last_clean":"2024-09-21T10:22:39.624342+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:22:39.624342+0000","last_undegraded":"2024-09-21T10:22:39.624342+0000","last_fullsized":"2024-09-21T10:22:39.624342+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":611,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.608887+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:22:49.608887+0000","last_peered":"2024-09-21T10:22:49.608887+0000","last_clean":"2024-09-21T10:22:49.608887+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:22:49.608887+0000","last_undegraded":"2024-09-21T10:22:49.608887+0000","last_fullsized":"2024-09-21T10:22:49.608887+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":412,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.612500+0000","last_change":"2024-09-21T10:22:01.331612+0000","last_active":"2024-09-21T10:22:49.612500+0000","last_peered":"2024-09-21T10:22:49.612500+0000","last_clean":"2024-09-21T10:22:49.612500+0000","last_became_active":"2024-09-21T10:22:01.325282+0000","last_became_peered":"2024-09-21T10:22:01.325282+0000","last_unstale":"2024-09-21T10:22:49.612500+0000","last_undegraded":"2024-09-21T10:22:49.612500+0000","last_fullsized":"2024-09-21T10:22:49.612500+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:23.383106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"357'9570","reported_seq":10634,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:59.982503+0000","last_change":"2024-09-21T10:21:39.510930+0000","last_active":"2024-09-21T10:22:59.982503+0000","last_peered":"2024-09-21T10:22:59.982503+0000","last_clean":"2024-09-21T10:20:12.056605+0000","last_became_active":"2024-09-21T10:20:14.466147+0000","last_became_peered":"2024-09-21T10:20:14.466147+0000","last_unstale":"2024-09-21T10:22:59.982503+0000","last_undegraded":"2024-09-21T10:20:14.094800+0000","last_fullsized":"2024-09-21T10:20:14.071992+0000","mapping_epoch":325,"log_start":"319'2200","ondisk_log_start":"319'2200","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7370,"log_dups_size":0,"ondisk_log_size":7370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:34.237367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193331200,"num_objects":2950,"num_object_clones":0,"num_object_copies":8850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3167,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9570,"num_write_kb":400640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":489,"num_bytes_recovered":32047104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-217},{"shards":"2,4,6","objects":345}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":46,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624853+0000","last_change":"2024-09-21T10:22:01.321467+0000","last_active":"2024-09-21T10:22:39.624853+0000","last_peered":"2024-09-21T10:22:39.624853+0000","last_clean":"2024-09-21T10:22:39.624853+0000","last_became_active":"2024-09-21T10:22:01.318270+0000","last_became_peered":"2024-09-21T10:22:01.318270+0000","last_unstale":"2024-09-21T10:22:39.624853+0000","last_undegraded":"2024-09-21T10:22:39.624853+0000","last_fullsized":"2024-09-21T10:22:39.624853+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:44.059553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":64,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.580407+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:22:39.580407+0000","last_peered":"2024-09-21T10:22:39.580407+0000","last_clean":"2024-09-21T10:22:39.580407+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:22:39.580407+0000","last_undegraded":"2024-09-21T10:22:39.580407+0000","last_fullsized":"2024-09-21T10:22:39.580407+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"357'9348","reported_seq":13220,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:59.939411+0000","last_change":"2024-09-21T10:20:04.684629+0000","last_active":"2024-09-21T10:22:59.939411+0000","last_peered":"2024-09-21T10:22:59.939411+0000","last_clean":"2024-09-21T10:22:59.939411+0000","last_became_active":"2024-09-21T10:20:04.680299+0000","last_became_peered":"2024-09-21T10:20:04.680299+0000","last_unstale":"2024-09-21T10:22:59.939411+0000","last_undegraded":"2024-09-21T10:22:59.939411+0000","last_fullsized":"2024-09-21T10:22:59.939411+0000","mapping_epoch":320,"log_start":"319'2800","ondisk_log_start":"319'2800","created":305,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:41.743081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190185472,"num_objects":2902,"num_object_clones":0,"num_object_copies":8706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9348,"num_write_kb":392000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1891,"num_bytes_recovered":123928576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":33,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624702+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:22:39.624702+0000","last_peered":"2024-09-21T10:22:39.624702+0000","last_clean":"2024-09-21T10:22:39.624702+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:22:39.624702+0000","last_undegraded":"2024-09-21T10:22:39.624702+0000","last_fullsized":"2024-09-21T10:22:39.624702+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":53,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.669751+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:22:39.669751+0000","last_peered":"2024-09-21T10:22:39.669751+0000","last_clean":"2024-09-21T10:22:39.669751+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:22:39.669751+0000","last_undegraded":"2024-09-21T10:22:39.669751+0000","last_fullsized":"2024-09-21T10:22:39.669751+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"357'9213","reported_seq":9329,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:56.620721+0000","last_change":"2024-09-21T10:19:22.649712+0000","last_active":"2024-09-21T10:22:56.620721+0000","last_peered":"2024-09-21T10:22:56.620721+0000","last_clean":"2024-09-21T10:22:56.620721+0000","last_became_active":"2024-09-21T10:19:22.649303+0000","last_became_peered":"2024-09-21T10:19:22.649303+0000","last_unstale":"2024-09-21T10:22:56.620721+0000","last_undegraded":"2024-09-21T10:22:56.620721+0000","last_fullsized":"2024-09-21T10:22:56.620721+0000","mapping_epoch":305,"log_start":"318'2061","ondisk_log_start":"318'2061","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7152,"log_dups_size":0,"ondisk_log_size":7152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:24.442851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202571776,"num_objects":3091,"num_object_clones":0,"num_object_copies":9273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9213,"num_write_kb":393728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":355,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.626105+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:22:39.626105+0000","last_peered":"2024-09-21T10:22:39.626105+0000","last_clean":"2024-09-21T10:22:39.626105+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:22:39.626105+0000","last_undegraded":"2024-09-21T10:22:39.626105+0000","last_fullsized":"2024-09-21T10:22:39.626105+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":375,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611679+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:22:49.611679+0000","last_peered":"2024-09-21T10:22:49.611679+0000","last_clean":"2024-09-21T10:22:49.611679+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:22:49.611679+0000","last_undegraded":"2024-09-21T10:22:49.611679+0000","last_fullsized":"2024-09-21T10:22:49.611679+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"357'9833","reported_seq":9949,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:23:00.381904+0000","last_change":"2024-09-21T10:19:22.640645+0000","last_active":"2024-09-21T10:23:00.381904+0000","last_peered":"2024-09-21T10:23:00.381904+0000","last_clean":"2024-09-21T10:23:00.381904+0000","last_became_active":"2024-09-21T10:19:22.640343+0000","last_became_peered":"2024-09-21T10:19:22.640343+0000","last_unstale":"2024-09-21T10:23:00.381904+0000","last_undegraded":"2024-09-21T10:23:00.381904+0000","last_fullsized":"2024-09-21T10:23:00.381904+0000","mapping_epoch":305,"log_start":"327'4500","ondisk_log_start":"327'4500","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5333,"log_dups_size":0,"ondisk_log_size":5333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:48.734122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197197824,"num_objects":3009,"num_object_clones":0,"num_object_copies":9027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9833,"num_write_kb":410944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":289,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.580763+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:22:39.580763+0000","last_peered":"2024-09-21T10:22:39.580763+0000","last_clean":"2024-09-21T10:22:39.580763+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:22:39.580763+0000","last_undegraded":"2024-09-21T10:22:39.580763+0000","last_fullsized":"2024-09-21T10:22:39.580763+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":202,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.616936+0000","last_change":"2024-09-21T10:19:50.585724+0000","last_active":"2024-09-21T10:22:49.616936+0000","last_peered":"2024-09-21T10:22:49.616936+0000","last_clean":"2024-09-21T10:22:49.616936+0000","last_became_active":"2024-09-21T10:19:50.585315+0000","last_became_peered":"2024-09-21T10:19:50.585315+0000","last_unstale":"2024-09-21T10:22:49.616936+0000","last_undegraded":"2024-09-21T10:22:49.616936+0000","last_fullsized":"2024-09-21T10:22:49.616936+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:26.618970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"357'9558","reported_seq":14637,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:58.697441+0000","last_change":"2024-09-21T10:22:01.328242+0000","last_active":"2024-09-21T10:22:58.697441+0000","last_peered":"2024-09-21T10:22:58.697441+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492359+0000","last_became_peered":"2024-09-21T10:20:14.492359+0000","last_unstale":"2024-09-21T10:22:58.697441+0000","last_undegraded":"2024-09-21T10:20:14.218670+0000","last_fullsized":"2024-09-21T10:20:14.071510+0000","mapping_epoch":325,"log_start":"319'2700","ondisk_log_start":"319'2700","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6858,"log_dups_size":0,"ondisk_log_size":6858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194379776,"num_objects":2966,"num_object_clones":0,"num_object_copies":8898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9558,"num_write_kb":400768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2481,"num_bytes_recovered":148307968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":1762}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":611,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624097+0000","last_change":"2024-09-21T10:22:04.029688+0000","last_active":"2024-09-21T10:22:39.624097+0000","last_peered":"2024-09-21T10:22:39.624097+0000","last_clean":"2024-09-21T10:22:39.624097+0000","last_became_active":"2024-09-21T10:22:04.028853+0000","last_became_peered":"2024-09-21T10:22:04.028853+0000","last_unstale":"2024-09-21T10:22:39.624097+0000","last_undegraded":"2024-09-21T10:22:39.624097+0000","last_fullsized":"2024-09-21T10:22:39.624097+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:20.457596+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:19:20.457596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":641,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.607676+0000","last_change":"2024-09-21T10:22:01.335303+0000","last_active":"2024-09-21T10:22:49.607676+0000","last_peered":"2024-09-21T10:22:49.607676+0000","last_clean":"2024-09-21T10:22:49.607676+0000","last_became_active":"2024-09-21T10:22:01.325298+0000","last_became_peered":"2024-09-21T10:22:01.325298+0000","last_unstale":"2024-09-21T10:22:49.607676+0000","last_undegraded":"2024-09-21T10:22:49.607676+0000","last_fullsized":"2024-09-21T10:22:49.607676+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:01.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":53,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.669847+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:22:39.669847+0000","last_peered":"2024-09-21T10:22:39.669847+0000","last_clean":"2024-09-21T10:22:39.669847+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:22:39.669847+0000","last_undegraded":"2024-09-21T10:22:39.669847+0000","last_fullsized":"2024-09-21T10:22:39.669847+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"357'9386","reported_seq":9502,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:23:00.252426+0000","last_change":"2024-09-21T10:19:22.641281+0000","last_active":"2024-09-21T10:23:00.252426+0000","last_peered":"2024-09-21T10:23:00.252426+0000","last_clean":"2024-09-21T10:23:00.252426+0000","last_became_active":"2024-09-21T10:19:22.640988+0000","last_became_peered":"2024-09-21T10:19:22.640988+0000","last_unstale":"2024-09-21T10:23:00.252426+0000","last_undegraded":"2024-09-21T10:23:00.252426+0000","last_fullsized":"2024-09-21T10:23:00.252426+0000","mapping_epoch":305,"log_start":"327'4100","ondisk_log_start":"327'4100","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5286,"log_dups_size":0,"ondisk_log_size":5286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:52.652565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184549376,"num_objects":2816,"num_object_clones":0,"num_object_copies":8448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9386,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":425,"reported_epoch":356,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:39.624196+0000","last_change":"2024-09-21T10:22:02.013601+0000","last_active":"2024-09-21T10:22:39.624196+0000","last_peered":"2024-09-21T10:22:39.624196+0000","last_clean":"2024-09-21T10:19:15.364615+0000","last_became_active":"2024-09-21T10:22:02.013140+0000","last_became_peered":"2024-09-21T10:22:02.013140+0000","last_unstale":"2024-09-21T10:22:39.624196+0000","last_undegraded":"2024-09-21T10:22:02.004849+0000","last_fullsized":"2024-09-21T10:22:02.002552+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_clean_scrub_stamp":"2024-09-21T10:19:15.365774+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1},{"shards":"6,7","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":365,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611791+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:22:49.611791+0000","last_peered":"2024-09-21T10:22:49.611791+0000","last_clean":"2024-09-21T10:22:49.611791+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:22:49.611791+0000","last_undegraded":"2024-09-21T10:22:49.611791+0000","last_fullsized":"2024-09-21T10:22:49.611791+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":53,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.670162+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:22:39.670162+0000","last_peered":"2024-09-21T10:22:39.670162+0000","last_clean":"2024-09-21T10:22:39.670162+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:22:39.670162+0000","last_undegraded":"2024-09-21T10:22:39.670162+0000","last_fullsized":"2024-09-21T10:22:39.670162+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"357'9607","reported_seq":16935,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:59.950036+0000","last_change":"2024-09-21T10:22:34.604937+0000","last_active":"2024-09-21T10:22:59.950036+0000","last_peered":"2024-09-21T10:22:59.950036+0000","last_clean":"2024-09-21T10:22:59.950036+0000","last_became_active":"2024-09-21T10:22:34.577215+0000","last_became_peered":"2024-09-21T10:22:34.577215+0000","last_unstale":"2024-09-21T10:22:59.950036+0000","last_undegraded":"2024-09-21T10:22:59.950036+0000","last_fullsized":"2024-09-21T10:22:59.950036+0000","mapping_epoch":354,"log_start":"319'2200","ondisk_log_start":"319'2200","created":305,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7407,"log_dups_size":0,"ondisk_log_size":7407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:40.287923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197066752,"num_objects":3007,"num_object_clones":0,"num_object_copies":9021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9607,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3618,"num_bytes_recovered":121634816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":53,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.669662+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:22:39.669662+0000","last_peered":"2024-09-21T10:22:39.669662+0000","last_clean":"2024-09-21T10:22:39.669662+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:22:39.669662+0000","last_undegraded":"2024-09-21T10:22:39.669662+0000","last_fullsized":"2024-09-21T10:22:39.669662+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":53,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.670358+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:22:39.670358+0000","last_peered":"2024-09-21T10:22:39.670358+0000","last_clean":"2024-09-21T10:22:39.670358+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:22:39.670358+0000","last_undegraded":"2024-09-21T10:22:39.670358+0000","last_fullsized":"2024-09-21T10:22:39.670358+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"357'9505","reported_seq":9634,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:00.396201+0000","last_change":"2024-09-21T10:21:39.558048+0000","last_active":"2024-09-21T10:23:00.396201+0000","last_peered":"2024-09-21T10:23:00.396201+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492538+0000","last_became_peered":"2024-09-21T10:20:14.492538+0000","last_unstale":"2024-09-21T10:23:00.396201+0000","last_undegraded":"2024-09-21T10:20:14.216051+0000","last_fullsized":"2024-09-21T10:20:14.075057+0000","mapping_epoch":325,"log_start":"327'4200","ondisk_log_start":"327'4200","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197722112,"num_objects":3017,"num_object_clones":0,"num_object_copies":9051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9505,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-559}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":611,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624385+0000","last_change":"2024-09-21T10:22:04.031320+0000","last_active":"2024-09-21T10:22:39.624385+0000","last_peered":"2024-09-21T10:22:39.624385+0000","last_clean":"2024-09-21T10:22:39.624385+0000","last_became_active":"2024-09-21T10:22:04.030774+0000","last_became_peered":"2024-09-21T10:22:04.030774+0000","last_unstale":"2024-09-21T10:22:39.624385+0000","last_undegraded":"2024-09-21T10:22:39.624385+0000","last_fullsized":"2024-09-21T10:22:39.624385+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:22.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:22.489745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":619,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606854+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:22:49.606854+0000","last_peered":"2024-09-21T10:22:49.606854+0000","last_clean":"2024-09-21T10:22:49.606854+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:22:49.606854+0000","last_undegraded":"2024-09-21T10:22:49.606854+0000","last_fullsized":"2024-09-21T10:22:49.606854+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":312,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.614468+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:22:49.614468+0000","last_peered":"2024-09-21T10:22:49.614468+0000","last_clean":"2024-09-21T10:22:49.614468+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:22:49.614468+0000","last_undegraded":"2024-09-21T10:22:49.614468+0000","last_fullsized":"2024-09-21T10:22:49.614468+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"357'9554","reported_seq":12716,"reported_epoch":357,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:22:59.939927+0000","last_change":"2024-09-21T10:22:02.106012+0000","last_active":"2024-09-21T10:22:59.939927+0000","last_peered":"2024-09-21T10:22:59.939927+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:22:02.086281+0000","last_became_peered":"2024-09-21T10:22:02.086281+0000","last_unstale":"2024-09-21T10:22:59.939927+0000","last_undegraded":"2024-09-21T10:22:02.081791+0000","last_fullsized":"2024-09-21T10:22:59.939927+0000","mapping_epoch":347,"log_start":"319'3000","ondisk_log_start":"319'3000","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:37.553227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189267968,"num_objects":2888,"num_object_clones":0,"num_object_copies":8664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2130,"num_objects_misplaced":2888,"num_objects_unfound":0,"num_objects_dirty":2888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9554,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1541,"num_bytes_recovered":100990976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,5,6","objects":2551},{"shards":"3,6","objects":337}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":611,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624124+0000","last_change":"2024-09-21T10:22:04.029284+0000","last_active":"2024-09-21T10:22:39.624124+0000","last_peered":"2024-09-21T10:22:39.624124+0000","last_clean":"2024-09-21T10:22:39.624124+0000","last_became_active":"2024-09-21T10:22:04.028938+0000","last_became_peered":"2024-09-21T10:22:04.028938+0000","last_unstale":"2024-09-21T10:22:39.624124+0000","last_undegraded":"2024-09-21T10:22:39.624124+0000","last_fullsized":"2024-09-21T10:22:39.624124+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:12.388304+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:12.388304+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":641,"reported_epoch":356,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:39.580691+0000","last_change":"2024-09-21T10:21:36.010162+0000","last_active":"2024-09-21T10:22:39.580691+0000","last_peered":"2024-09-21T10:22:39.580691+0000","last_clean":"2024-09-21T10:21:17.725138+0000","last_became_active":"2024-09-21T10:21:36.009706+0000","last_became_peered":"2024-09-21T10:21:36.009706+0000","last_unstale":"2024-09-21T10:22:39.580691+0000","last_undegraded":"2024-09-21T10:21:35.852171+0000","last_fullsized":"2024-09-21T10:21:35.851526+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:04.206298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":551,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606455+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:22:49.606455+0000","last_peered":"2024-09-21T10:22:49.606455+0000","last_clean":"2024-09-21T10:22:49.606455+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:22:49.606455+0000","last_undegraded":"2024-09-21T10:22:49.606455+0000","last_fullsized":"2024-09-21T10:22:49.606455+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":331,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.584284+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:22:39.584284+0000","last_peered":"2024-09-21T10:22:39.584284+0000","last_clean":"2024-09-21T10:22:39.584284+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:22:39.584284+0000","last_undegraded":"2024-09-21T10:22:39.584284+0000","last_fullsized":"2024-09-21T10:22:39.584284+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"357'9340","reported_seq":9456,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:00.401028+0000","last_change":"2024-09-21T10:20:14.480108+0000","last_active":"2024-09-21T10:23:00.401028+0000","last_peered":"2024-09-21T10:23:00.401028+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.471864+0000","last_became_peered":"2024-09-21T10:20:14.471864+0000","last_unstale":"2024-09-21T10:23:00.401028+0000","last_undegraded":"2024-09-21T10:20:14.214064+0000","last_fullsized":"2024-09-21T10:20:14.074889+0000","mapping_epoch":325,"log_start":"327'4000","ondisk_log_start":"327'4000","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5340,"log_dups_size":0,"ondisk_log_size":5340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187695104,"num_objects":2864,"num_object_clones":0,"num_object_copies":8592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9340,"num_write_kb":390528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-681}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":611,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624453+0000","last_change":"2024-09-21T10:22:04.031573+0000","last_active":"2024-09-21T10:22:39.624453+0000","last_peered":"2024-09-21T10:22:39.624453+0000","last_clean":"2024-09-21T10:22:39.624453+0000","last_became_active":"2024-09-21T10:22:04.031125+0000","last_became_peered":"2024-09-21T10:22:04.031125+0000","last_unstale":"2024-09-21T10:22:39.624453+0000","last_undegraded":"2024-09-21T10:22:39.624453+0000","last_fullsized":"2024-09-21T10:22:39.624453+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:19:19.450903+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":996,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624308+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:22:39.624308+0000","last_peered":"2024-09-21T10:22:39.624308+0000","last_clean":"2024-09-21T10:22:39.624308+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:22:39.624308+0000","last_undegraded":"2024-09-21T10:22:39.624308+0000","last_fullsized":"2024-09-21T10:22:39.624308+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":201,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.618398+0000","last_change":"2024-09-21T10:19:51.555124+0000","last_active":"2024-09-21T10:22:49.618398+0000","last_peered":"2024-09-21T10:22:49.618398+0000","last_clean":"2024-09-21T10:22:49.618398+0000","last_became_active":"2024-09-21T10:19:51.554766+0000","last_became_peered":"2024-09-21T10:19:51.554766+0000","last_unstale":"2024-09-21T10:22:49.618398+0000","last_undegraded":"2024-09-21T10:22:49.618398+0000","last_fullsized":"2024-09-21T10:22:49.618398+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:52.832950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":32,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.617562+0000","last_change":"2024-09-21T10:22:01.316140+0000","last_active":"2024-09-21T10:22:49.617562+0000","last_peered":"2024-09-21T10:22:49.617562+0000","last_clean":"2024-09-21T10:22:49.617562+0000","last_became_active":"2024-09-21T10:22:01.315408+0000","last_became_peered":"2024-09-21T10:22:01.315408+0000","last_unstale":"2024-09-21T10:22:49.617562+0000","last_undegraded":"2024-09-21T10:22:49.617562+0000","last_fullsized":"2024-09-21T10:22:49.617562+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:07.322279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"357'9543","reported_seq":13520,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:58.748618+0000","last_change":"2024-09-21T10:20:47.589372+0000","last_active":"2024-09-21T10:22:58.748618+0000","last_peered":"2024-09-21T10:22:58.748618+0000","last_clean":"2024-09-21T10:22:58.748618+0000","last_became_active":"2024-09-21T10:20:47.580051+0000","last_became_peered":"2024-09-21T10:20:47.580051+0000","last_unstale":"2024-09-21T10:22:58.748618+0000","last_undegraded":"2024-09-21T10:22:58.748618+0000","last_fullsized":"2024-09-21T10:22:58.748618+0000","mapping_epoch":332,"log_start":"319'2700","ondisk_log_start":"319'2700","created":305,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6843,"log_dups_size":0,"ondisk_log_size":6843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:46.363318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200212480,"num_objects":3055,"num_object_clones":0,"num_object_copies":9165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9543,"num_write_kb":403136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":611,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624207+0000","last_change":"2024-09-21T10:22:04.030341+0000","last_active":"2024-09-21T10:22:39.624207+0000","last_peered":"2024-09-21T10:22:39.624207+0000","last_clean":"2024-09-21T10:22:39.624207+0000","last_became_active":"2024-09-21T10:22:04.030000+0000","last_became_peered":"2024-09-21T10:22:04.030000+0000","last_unstale":"2024-09-21T10:22:39.624207+0000","last_undegraded":"2024-09-21T10:22:39.624207+0000","last_fullsized":"2024-09-21T10:22:39.624207+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:10.332770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:10.332770+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1140,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.617324+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:22:49.617324+0000","last_peered":"2024-09-21T10:22:49.617324+0000","last_clean":"2024-09-21T10:22:49.617324+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:22:49.617324+0000","last_undegraded":"2024-09-21T10:22:49.617324+0000","last_fullsized":"2024-09-21T10:22:49.617324+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":298,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.613096+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:22:49.613096+0000","last_peered":"2024-09-21T10:22:49.613096+0000","last_clean":"2024-09-21T10:22:49.613096+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:22:49.613096+0000","last_undegraded":"2024-09-21T10:22:49.613096+0000","last_fullsized":"2024-09-21T10:22:49.613096+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":328,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.614600+0000","last_change":"2024-09-21T10:22:01.317933+0000","last_active":"2024-09-21T10:22:49.614600+0000","last_peered":"2024-09-21T10:22:49.614600+0000","last_clean":"2024-09-21T10:22:49.614600+0000","last_became_active":"2024-09-21T10:22:01.317536+0000","last_became_peered":"2024-09-21T10:22:01.317536+0000","last_unstale":"2024-09-21T10:22:49.614600+0000","last_undegraded":"2024-09-21T10:22:49.614600+0000","last_fullsized":"2024-09-21T10:22:49.614600+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:34.312724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"357'9574","reported_seq":16776,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:58.817296+0000","last_change":"2024-09-21T10:21:34.868559+0000","last_active":"2024-09-21T10:22:58.817296+0000","last_peered":"2024-09-21T10:22:58.817296+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.463996+0000","last_became_peered":"2024-09-21T10:20:14.463996+0000","last_unstale":"2024-09-21T10:22:58.817296+0000","last_undegraded":"2024-09-21T10:20:14.217266+0000","last_fullsized":"2024-09-21T10:20:14.071353+0000","mapping_epoch":325,"log_start":"319'2700","ondisk_log_start":"319'2700","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6874,"log_dups_size":0,"ondisk_log_size":6874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202899456,"num_objects":3096,"num_object_clones":0,"num_object_copies":9288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":98,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9574,"num_write_kb":405440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":232128512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2998}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":611,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624191+0000","last_change":"2024-09-21T10:22:04.029892+0000","last_active":"2024-09-21T10:22:39.624191+0000","last_peered":"2024-09-21T10:22:39.624191+0000","last_clean":"2024-09-21T10:22:39.624191+0000","last_became_active":"2024-09-21T10:22:04.029353+0000","last_became_peered":"2024-09-21T10:22:04.029353+0000","last_unstale":"2024-09-21T10:22:39.624191+0000","last_undegraded":"2024-09-21T10:22:39.624191+0000","last_fullsized":"2024-09-21T10:22:39.624191+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:19:13.371825+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1099,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.580564+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:22:39.580564+0000","last_peered":"2024-09-21T10:22:39.580564+0000","last_clean":"2024-09-21T10:22:39.580564+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:22:39.580564+0000","last_undegraded":"2024-09-21T10:22:39.580564+0000","last_fullsized":"2024-09-21T10:22:39.580564+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":53,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.670258+0000","last_change":"2024-09-21T10:19:50.581529+0000","last_active":"2024-09-21T10:22:39.670258+0000","last_peered":"2024-09-21T10:22:39.670258+0000","last_clean":"2024-09-21T10:22:39.670258+0000","last_became_active":"2024-09-21T10:19:50.581119+0000","last_became_peered":"2024-09-21T10:19:50.581119+0000","last_unstale":"2024-09-21T10:22:39.670258+0000","last_undegraded":"2024-09-21T10:22:39.670258+0000","last_fullsized":"2024-09-21T10:22:39.670258+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:13.334483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":298,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611243+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:22:49.611243+0000","last_peered":"2024-09-21T10:22:49.611243+0000","last_clean":"2024-09-21T10:22:49.611243+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:22:49.611243+0000","last_undegraded":"2024-09-21T10:22:49.611243+0000","last_fullsized":"2024-09-21T10:22:49.611243+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"357'9626","reported_seq":13731,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:56.615965+0000","last_change":"2024-09-21T10:20:42.601607+0000","last_active":"2024-09-21T10:22:56.615965+0000","last_peered":"2024-09-21T10:22:56.615965+0000","last_clean":"2024-09-21T10:22:56.615965+0000","last_became_active":"2024-09-21T10:20:42.593406+0000","last_became_peered":"2024-09-21T10:20:42.593406+0000","last_unstale":"2024-09-21T10:22:56.615965+0000","last_undegraded":"2024-09-21T10:22:56.615965+0000","last_fullsized":"2024-09-21T10:22:56.615965+0000","mapping_epoch":330,"log_start":"319'2468","ondisk_log_start":"319'2468","created":305,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7158,"log_dups_size":0,"ondisk_log_size":7158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:09.770489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202899456,"num_objects":3096,"num_object_clones":0,"num_object_copies":9288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9626,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1985,"num_bytes_recovered":130088960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":611,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624898+0000","last_change":"2024-09-21T10:22:04.029371+0000","last_active":"2024-09-21T10:22:39.624898+0000","last_peered":"2024-09-21T10:22:39.624898+0000","last_clean":"2024-09-21T10:22:39.624898+0000","last_became_active":"2024-09-21T10:22:04.029014+0000","last_became_peered":"2024-09-21T10:22:04.029014+0000","last_unstale":"2024-09-21T10:22:39.624898+0000","last_undegraded":"2024-09-21T10:22:39.624898+0000","last_fullsized":"2024-09-21T10:22:39.624898+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":304,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.611757+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:22:39.611757+0000","last_peered":"2024-09-21T10:22:39.611757+0000","last_clean":"2024-09-21T10:22:39.611757+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:22:39.611757+0000","last_undegraded":"2024-09-21T10:22:39.611757+0000","last_fullsized":"2024-09-21T10:22:39.611757+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":907,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611467+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:22:49.611467+0000","last_peered":"2024-09-21T10:22:49.611467+0000","last_clean":"2024-09-21T10:22:49.611467+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:22:49.611467+0000","last_undegraded":"2024-09-21T10:22:49.611467+0000","last_fullsized":"2024-09-21T10:22:49.611467+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":334,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.619241+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:22:49.619241+0000","last_peered":"2024-09-21T10:22:49.619241+0000","last_clean":"2024-09-21T10:22:49.619241+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:22:49.619241+0000","last_undegraded":"2024-09-21T10:22:49.619241+0000","last_fullsized":"2024-09-21T10:22:49.619241+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"357'9429","reported_seq":9545,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:58.778945+0000","last_change":"2024-09-21T10:19:22.641043+0000","last_active":"2024-09-21T10:22:58.778945+0000","last_peered":"2024-09-21T10:22:58.778945+0000","last_clean":"2024-09-21T10:22:58.778945+0000","last_became_active":"2024-09-21T10:19:22.640742+0000","last_became_peered":"2024-09-21T10:19:22.640742+0000","last_unstale":"2024-09-21T10:22:58.778945+0000","last_undegraded":"2024-09-21T10:22:58.778945+0000","last_fullsized":"2024-09-21T10:22:58.778945+0000","mapping_epoch":305,"log_start":"319'2600","ondisk_log_start":"319'2600","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6829,"log_dups_size":0,"ondisk_log_size":6829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:36.901208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191168512,"num_objects":2917,"num_object_clones":0,"num_object_copies":8751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9429,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":611,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.624780+0000","last_change":"2024-09-21T10:22:04.031872+0000","last_active":"2024-09-21T10:22:39.624780+0000","last_peered":"2024-09-21T10:22:39.624780+0000","last_clean":"2024-09-21T10:22:39.624780+0000","last_became_active":"2024-09-21T10:22:04.031343+0000","last_became_peered":"2024-09-21T10:22:04.031343+0000","last_unstale":"2024-09-21T10:22:39.624780+0000","last_undegraded":"2024-09-21T10:22:39.624780+0000","last_fullsized":"2024-09-21T10:22:39.624780+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:07.332404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:07.332404+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":316,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611384+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:22:49.611384+0000","last_peered":"2024-09-21T10:22:49.611384+0000","last_clean":"2024-09-21T10:22:49.611384+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:22:49.611384+0000","last_undegraded":"2024-09-21T10:22:49.611384+0000","last_fullsized":"2024-09-21T10:22:49.611384+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":923,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:44.875350+0000","last_change":"2024-09-21T10:22:01.336961+0000","last_active":"2024-09-21T10:22:44.875350+0000","last_peered":"2024-09-21T10:22:44.875350+0000","last_clean":"2024-09-21T10:22:44.875350+0000","last_became_active":"2024-09-21T10:22:01.320079+0000","last_became_peered":"2024-09-21T10:22:01.320079+0000","last_unstale":"2024-09-21T10:22:44.875350+0000","last_undegraded":"2024-09-21T10:22:44.875350+0000","last_fullsized":"2024-09-21T10:22:44.875350+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:54.037388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":200,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.616835+0000","last_change":"2024-09-21T10:20:13.088557+0000","last_active":"2024-09-21T10:22:49.616835+0000","last_peered":"2024-09-21T10:22:49.616835+0000","last_clean":"2024-09-21T10:22:49.616835+0000","last_became_active":"2024-09-21T10:20:13.088123+0000","last_became_peered":"2024-09-21T10:20:13.088123+0000","last_unstale":"2024-09-21T10:22:49.616835+0000","last_undegraded":"2024-09-21T10:22:49.616835+0000","last_fullsized":"2024-09-21T10:22:49.616835+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:49.475499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"357'9601","reported_seq":13559,"reported_epoch":357,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:22:58.837176+0000","last_change":"2024-09-21T10:22:01.405956+0000","last_active":"2024-09-21T10:22:58.837176+0000","last_peered":"2024-09-21T10:22:58.837176+0000","last_clean":"2024-09-21T10:20:12.067444+0000","last_became_active":"2024-09-21T10:22:01.391808+0000","last_became_peered":"2024-09-21T10:22:01.391808+0000","last_unstale":"2024-09-21T10:22:58.837176+0000","last_undegraded":"2024-09-21T10:22:01.123056+0000","last_fullsized":"2024-09-21T10:22:58.837176+0000","mapping_epoch":346,"log_start":"319'2700","ondisk_log_start":"319'2700","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6901,"log_dups_size":0,"ondisk_log_size":6901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193658880,"num_objects":2955,"num_object_clones":0,"num_object_copies":8865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1803,"num_objects_misplaced":4107,"num_objects_unfound":0,"num_objects_dirty":2955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9601,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1910,"num_bytes_recovered":125042688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[3,5,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,5","objects":2565},{"shards":"3,4,6","objects":390}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":611,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.626031+0000","last_change":"2024-09-21T10:22:04.032207+0000","last_active":"2024-09-21T10:22:39.626031+0000","last_peered":"2024-09-21T10:22:39.626031+0000","last_clean":"2024-09-21T10:22:39.626031+0000","last_became_active":"2024-09-21T10:22:04.031881+0000","last_became_peered":"2024-09-21T10:22:04.031881+0000","last_unstale":"2024-09-21T10:22:39.626031+0000","last_undegraded":"2024-09-21T10:22:39.626031+0000","last_fullsized":"2024-09-21T10:22:39.626031+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:08.332410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:08.332410+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":624,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.609249+0000","last_change":"2024-09-21T10:21:34.876196+0000","last_active":"2024-09-21T10:22:49.609249+0000","last_peered":"2024-09-21T10:22:49.609249+0000","last_clean":"2024-09-21T10:22:49.609249+0000","last_became_active":"2024-09-21T10:21:34.868031+0000","last_became_peered":"2024-09-21T10:21:34.868031+0000","last_unstale":"2024-09-21T10:22:49.609249+0000","last_undegraded":"2024-09-21T10:22:49.609249+0000","last_fullsized":"2024-09-21T10:22:49.609249+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:58.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":970,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.612355+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:22:49.612355+0000","last_peered":"2024-09-21T10:22:49.612355+0000","last_clean":"2024-09-21T10:22:49.612355+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:22:49.612355+0000","last_undegraded":"2024-09-21T10:22:49.612355+0000","last_fullsized":"2024-09-21T10:22:49.612355+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":620,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606694+0000","last_change":"2024-09-21T10:22:01.329400+0000","last_active":"2024-09-21T10:22:49.606694+0000","last_peered":"2024-09-21T10:22:49.606694+0000","last_clean":"2024-09-21T10:22:49.606694+0000","last_became_active":"2024-09-21T10:22:01.328887+0000","last_became_peered":"2024-09-21T10:22:01.328887+0000","last_unstale":"2024-09-21T10:22:49.606694+0000","last_undegraded":"2024-09-21T10:22:49.606694+0000","last_fullsized":"2024-09-21T10:22:49.606694+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:09.626637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"357'9488","reported_seq":13306,"reported_epoch":357,"state":"active+recovering+degraded+remapped","last_fresh":"2024-09-21T10:22:59.885389+0000","last_change":"2024-09-21T10:21:39.570426+0000","last_active":"2024-09-21T10:22:59.885389+0000","last_peered":"2024-09-21T10:22:59.885389+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:21:39.424206+0000","last_became_peered":"2024-09-21T10:21:39.424206+0000","last_unstale":"2024-09-21T10:22:59.885389+0000","last_undegraded":"2024-09-21T10:21:39.368782+0000","last_fullsized":"2024-09-21T10:22:59.885389+0000","mapping_epoch":339,"log_start":"313'300","ondisk_log_start":"313'300","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":9188,"log_dups_size":0,"ondisk_log_size":9188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:54.426685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188809240,"num_objects":2882,"num_object_clones":0,"num_object_copies":8646,"num_objects_missing_on_primary":1796,"num_objects_missing":1796,"num_objects_degraded":349,"num_objects_misplaced":355,"num_objects_unfound":0,"num_objects_dirty":2882,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9488,"num_write_kb":395777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1856,"num_bytes_recovered":121372672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,5,6","objects":2527},{"shards":"2,6","objects":349}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":586,"reported_epoch":356,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:39.624619+0000","last_change":"2024-09-21T10:22:02.009815+0000","last_active":"2024-09-21T10:22:39.624619+0000","last_peered":"2024-09-21T10:22:39.624619+0000","last_clean":"2024-09-21T10:12:59.519603+0000","last_became_active":"2024-09-21T10:22:02.009463+0000","last_became_peered":"2024-09-21T10:22:02.009463+0000","last_unstale":"2024-09-21T10:22:39.624619+0000","last_undegraded":"2024-09-21T10:22:02.002510+0000","last_fullsized":"2024-09-21T10:22:02.001835+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:02.435487+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:02.435487+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1},{"shards":"6,7","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":29,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:49.617664+0000","last_change":"2024-09-21T10:22:01.318966+0000","last_active":"2024-09-21T10:22:49.617664+0000","last_peered":"2024-09-21T10:22:49.617664+0000","last_clean":"2024-09-21T10:21:59.002104+0000","last_became_active":"2024-09-21T10:22:01.317312+0000","last_became_peered":"2024-09-21T10:22:01.317312+0000","last_unstale":"2024-09-21T10:22:49.617664+0000","last_undegraded":"2024-09-21T10:22:00.997012+0000","last_fullsized":"2024-09-21T10:22:49.617664+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":642,"reported_epoch":356,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:39.583768+0000","last_change":"2024-09-21T10:21:34.866048+0000","last_active":"2024-09-21T10:22:39.583768+0000","last_peered":"2024-09-21T10:22:39.583768+0000","last_clean":"2024-09-21T10:21:17.726080+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:22:39.583768+0000","last_undegraded":"2024-09-21T10:21:34.856315+0000","last_fullsized":"2024-09-21T10:22:39.583768+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":329,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.669855+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:22:39.669855+0000","last_peered":"2024-09-21T10:22:39.669855+0000","last_clean":"2024-09-21T10:22:39.669855+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:22:39.669855+0000","last_undegraded":"2024-09-21T10:22:39.669855+0000","last_fullsized":"2024-09-21T10:22:39.669855+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":353,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.670282+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:22:39.670282+0000","last_peered":"2024-09-21T10:22:39.670282+0000","last_clean":"2024-09-21T10:22:39.670282+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:22:39.670282+0000","last_undegraded":"2024-09-21T10:22:39.670282+0000","last_fullsized":"2024-09-21T10:22:39.670282+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":63,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.669936+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:22:39.669936+0000","last_peered":"2024-09-21T10:22:39.669936+0000","last_clean":"2024-09-21T10:22:39.669936+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:22:39.669936+0000","last_undegraded":"2024-09-21T10:22:39.669936+0000","last_fullsized":"2024-09-21T10:22:39.669936+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.580592+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:22:39.580592+0000","last_peered":"2024-09-21T10:22:39.580592+0000","last_clean":"2024-09-21T10:22:39.580592+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:22:39.580592+0000","last_undegraded":"2024-09-21T10:22:39.580592+0000","last_fullsized":"2024-09-21T10:22:39.580592+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":35,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.625951+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:22:39.625951+0000","last_peered":"2024-09-21T10:22:39.625951+0000","last_clean":"2024-09-21T10:22:39.625951+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:22:39.625951+0000","last_undegraded":"2024-09-21T10:22:39.625951+0000","last_fullsized":"2024-09-21T10:22:39.625951+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":64,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.580636+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:22:39.580636+0000","last_peered":"2024-09-21T10:22:39.580636+0000","last_clean":"2024-09-21T10:22:39.580636+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:22:39.580636+0000","last_undegraded":"2024-09-21T10:22:39.580636+0000","last_fullsized":"2024-09-21T10:22:39.580636+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":544,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606617+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:22:49.606617+0000","last_peered":"2024-09-21T10:22:49.606617+0000","last_clean":"2024-09-21T10:22:49.606617+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:22:49.606617+0000","last_undegraded":"2024-09-21T10:22:49.606617+0000","last_fullsized":"2024-09-21T10:22:49.606617+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":199,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.617768+0000","last_change":"2024-09-21T10:20:13.089473+0000","last_active":"2024-09-21T10:22:49.617768+0000","last_peered":"2024-09-21T10:22:49.617768+0000","last_clean":"2024-09-21T10:22:49.617768+0000","last_became_active":"2024-09-21T10:20:13.089111+0000","last_became_peered":"2024-09-21T10:20:13.089111+0000","last_unstale":"2024-09-21T10:22:49.617768+0000","last_undegraded":"2024-09-21T10:22:49.617768+0000","last_fullsized":"2024-09-21T10:22:49.617768+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:06.603284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":604,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606613+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:22:49.606613+0000","last_peered":"2024-09-21T10:22:49.606613+0000","last_clean":"2024-09-21T10:22:49.606613+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:22:49.606613+0000","last_undegraded":"2024-09-21T10:22:49.606613+0000","last_fullsized":"2024-09-21T10:22:49.606613+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":370,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.584543+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:22:39.584543+0000","last_peered":"2024-09-21T10:22:39.584543+0000","last_clean":"2024-09-21T10:22:39.584543+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:22:39.584543+0000","last_undegraded":"2024-09-21T10:22:39.584543+0000","last_fullsized":"2024-09-21T10:22:39.584543+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":620,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606775+0000","last_change":"2024-09-21T10:22:01.329690+0000","last_active":"2024-09-21T10:22:49.606775+0000","last_peered":"2024-09-21T10:22:49.606775+0000","last_clean":"2024-09-21T10:22:49.606775+0000","last_became_active":"2024-09-21T10:22:01.329033+0000","last_became_peered":"2024-09-21T10:22:01.329033+0000","last_unstale":"2024-09-21T10:22:49.606775+0000","last_undegraded":"2024-09-21T10:22:49.606775+0000","last_fullsized":"2024-09-21T10:22:49.606775+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:24.559524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":203,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.618618+0000","last_change":"2024-09-21T10:21:34.870704+0000","last_active":"2024-09-21T10:22:49.618618+0000","last_peered":"2024-09-21T10:22:49.618618+0000","last_clean":"2024-09-21T10:22:49.618618+0000","last_became_active":"2024-09-21T10:21:34.869835+0000","last_became_peered":"2024-09-21T10:21:34.869835+0000","last_unstale":"2024-09-21T10:22:49.618618+0000","last_undegraded":"2024-09-21T10:22:49.618618+0000","last_fullsized":"2024-09-21T10:22:49.618618+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:34.959915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":319,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.613541+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:22:49.613541+0000","last_peered":"2024-09-21T10:22:49.613541+0000","last_clean":"2024-09-21T10:22:49.613541+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:22:49.613541+0000","last_undegraded":"2024-09-21T10:22:49.613541+0000","last_fullsized":"2024-09-21T10:22:49.613541+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":610,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606717+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:22:49.606717+0000","last_peered":"2024-09-21T10:22:49.606717+0000","last_clean":"2024-09-21T10:22:49.606717+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:22:49.606717+0000","last_undegraded":"2024-09-21T10:22:49.606717+0000","last_fullsized":"2024-09-21T10:22:49.606717+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":361,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606785+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:22:49.606785+0000","last_peered":"2024-09-21T10:22:49.606785+0000","last_clean":"2024-09-21T10:22:49.606785+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:22:49.606785+0000","last_undegraded":"2024-09-21T10:22:49.606785+0000","last_fullsized":"2024-09-21T10:22:49.606785+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":609,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.617639+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:22:49.617639+0000","last_peered":"2024-09-21T10:22:49.617639+0000","last_clean":"2024-09-21T10:22:49.617639+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:22:49.617639+0000","last_undegraded":"2024-09-21T10:22:49.617639+0000","last_fullsized":"2024-09-21T10:22:49.617639+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":32,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.616619+0000","last_change":"2024-09-21T10:22:01.318549+0000","last_active":"2024-09-21T10:22:49.616619+0000","last_peered":"2024-09-21T10:22:49.616619+0000","last_clean":"2024-09-21T10:22:49.616619+0000","last_became_active":"2024-09-21T10:22:01.318071+0000","last_became_peered":"2024-09-21T10:22:01.318071+0000","last_unstale":"2024-09-21T10:22:49.616619+0000","last_undegraded":"2024-09-21T10:22:49.616619+0000","last_fullsized":"2024-09-21T10:22:49.616619+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:49.061492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":53,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.670146+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:22:39.670146+0000","last_peered":"2024-09-21T10:22:39.670146+0000","last_clean":"2024-09-21T10:22:39.670146+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:22:39.670146+0000","last_undegraded":"2024-09-21T10:22:39.670146+0000","last_fullsized":"2024-09-21T10:22:39.670146+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":296,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.811606+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:22:39.811606+0000","last_peered":"2024-09-21T10:22:39.811606+0000","last_clean":"2024-09-21T10:22:39.811606+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:22:39.811606+0000","last_undegraded":"2024-09-21T10:22:39.811606+0000","last_fullsized":"2024-09-21T10:22:39.811606+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":53,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.669789+0000","last_change":"2024-09-21T10:19:50.588363+0000","last_active":"2024-09-21T10:22:39.669789+0000","last_peered":"2024-09-21T10:22:39.669789+0000","last_clean":"2024-09-21T10:22:39.669789+0000","last_became_active":"2024-09-21T10:19:50.588107+0000","last_became_peered":"2024-09-21T10:19:50.588107+0000","last_unstale":"2024-09-21T10:22:39.669789+0000","last_undegraded":"2024-09-21T10:22:39.669789+0000","last_fullsized":"2024-09-21T10:22:39.669789+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:05.698018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":274,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.812436+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:22:39.812436+0000","last_peered":"2024-09-21T10:22:39.812436+0000","last_clean":"2024-09-21T10:22:39.812436+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:22:39.812436+0000","last_undegraded":"2024-09-21T10:22:39.812436+0000","last_fullsized":"2024-09-21T10:22:39.812436+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":623,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.607210+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:22:49.607210+0000","last_peered":"2024-09-21T10:22:49.607210+0000","last_clean":"2024-09-21T10:22:49.607210+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:22:49.607210+0000","last_undegraded":"2024-09-21T10:22:49.607210+0000","last_fullsized":"2024-09-21T10:22:49.607210+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":585,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.609165+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:22:49.609165+0000","last_peered":"2024-09-21T10:22:49.609165+0000","last_clean":"2024-09-21T10:22:49.609165+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:22:49.609165+0000","last_undegraded":"2024-09-21T10:22:49.609165+0000","last_fullsized":"2024-09-21T10:22:49.609165+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":53,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-21T10:22:39.669648+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:22:39.669648+0000","last_peered":"2024-09-21T10:22:39.669648+0000","last_clean":"2024-09-21T10:22:39.669648+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:22:39.669648+0000","last_undegraded":"2024-09-21T10:22:39.669648+0000","last_fullsized":"2024-09-21T10:22:39.669648+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":29,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:49.617555+0000","last_change":"2024-09-21T10:22:01.317167+0000","last_active":"2024-09-21T10:22:49.617555+0000","last_peered":"2024-09-21T10:22:49.617555+0000","last_clean":"2024-09-21T10:14:43.015500+0000","last_became_active":"2024-09-21T10:22:01.316569+0000","last_became_peered":"2024-09-21T10:22:01.316569+0000","last_unstale":"2024-09-21T10:22:49.617555+0000","last_undegraded":"2024-09-21T10:22:00.997069+0000","last_fullsized":"2024-09-21T10:22:49.617555+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3113615384,"num_objects":47511,"num_object_clones":0,"num_object_copies":142533,"num_objects_missing_on_primary":1796,"num_objects_missing":1796,"num_objects_degraded":15872,"num_objects_misplaced":7350,"num_objects_unfound":0,"num_objects_dirty":47511,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152175,"num_write_kb":6389889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21237,"num_bytes_recovered":1261633536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9895227392,"data_stored":9895215176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106946,"ondisk_log_size":106946,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":3,"num_objects_missing":3,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24225,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":391,"num_read_kb":833,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":3107188,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1640,"ondisk_log_size":1640,"up":33,"acting":31,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234400,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1177784,"kb_used_data":583088,"kb_used_omap":1,"kb_used_meta":594686,"kb_avail":92567368,"statfs":{"total":95995035648,"available":94788984832,"internally_reserved":0,"allocated":597082112,"data_stored":595248644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":608958923},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496038,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2232320,"kb_used_data":1615480,"kb_used_omap":1,"kb_used_meta":616830,"kb_avail":91512832,"statfs":{"total":95995035648,"available":93709139968,"internally_reserved":0,"allocated":1654251520,"data_stored":1652411214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":631634375},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403938,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1810672,"kb_used_data":1570344,"kb_used_omap":7,"kb_used_meta":240312,"kb_avail":91934480,"statfs":{"total":95995035648,"available":94140907520,"internally_reserved":0,"allocated":1608032256,"data_stored":1606203034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":246080408},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,1,1,1,1,2,1,5,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019311,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1739640,"kb_used_data":1149040,"kb_used_omap":1,"kb_used_meta":590590,"kb_avail":92005512,"statfs":{"total":95995035648,"available":94213644288,"internally_reserved":0,"allocated":1176616960,"data_stored":1174783753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":604764614},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435495,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2363684,"kb_used_data":1583708,"kb_used_omap":5,"kb_used_meta":779962,"kb_avail":91381468,"statfs":{"total":95995035648,"available":93574623232,"internally_reserved":0,"allocated":1621716992,"data_stored":1619899465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":798681128},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599022,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1816300,"kb_used_data":1126564,"kb_used_omap":7,"kb_used_meta":689720,"kb_avail":91928852,"statfs":{"total":95995035648,"available":94135144448,"internally_reserved":0,"allocated":1153601536,"data_stored":1151776603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":706273741},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762549,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1931944,"kb_used_data":1332384,"kb_used_omap":7,"kb_used_meta":599480,"kb_avail":91813208,"statfs":{"total":95995035648,"available":94016724992,"internally_reserved":0,"allocated":1364361216,"data_stored":1362539815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":613867773},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926075,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1262052,"kb_used_data":801372,"kb_used_omap":7,"kb_used_meta":460664,"kb_avail":92483100,"statfs":{"total":95995035648,"available":94702694400,"internally_reserved":0,"allocated":820604928,"data_stored":818787333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":471720126},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":802816000,"data_stored":802816000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1343225856,"data_stored":1343225856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1147408384,"data_stored":1147404312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1613037568,"data_stored":1613037568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1162280960,"data_stored":1162280960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1598492672,"data_stored":1598488600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1642532864,"data_stored":1642528792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":585433088,"data_stored":585433088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:23:03.212 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:03.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:03.314 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:03.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:03.448 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:03.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.484 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:03.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:03.586 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:03.615 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:03.717 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:03.747 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:03.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:03.849 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:03.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:03.984 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.016 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:04.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:04.117 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:04.141 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:04.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:03 smithi032 ceph-mon[105748]: pgmap v1072: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 15897/141375 objects degraded (11.245%); 7350/141375 objects misplaced (5.199%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:23:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:03 smithi032 ceph-mon[101299]: pgmap v1072: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 15897/141375 objects degraded (11.245%); 7350/141375 objects misplaced (5.199%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:23:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:04.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.278 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:04.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:03 smithi045 ceph-mon[95976]: pgmap v1072: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 15897/141375 objects degraded (11.245%); 7350/141375 objects misplaced (5.199%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:23:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:04.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:04.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.413 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:04.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:04.514 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:04.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.547 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:04.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:04.648 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:04.680 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:04.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:23:04.782 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:04.818 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:04.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:04.920 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:04.952 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:05.053 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:05.091 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:05.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:05.193 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:05.229 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:05.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:05.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:05.364 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:05.466 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:05.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:05.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:05.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:05.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.668 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:05.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:05.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:05.816 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:05.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:05.918 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:05.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.966 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:05.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:06.067 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:06.101 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:06.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:06.135 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:05 smithi032 ceph-mon[105748]: pgmap v1073: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 15686/138402 objects degraded (11.334%); 7139/138402 objects misplaced (5.158%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:23:06.135 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:05 smithi032 ceph-mon[101299]: pgmap v1073: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 15686/138402 objects degraded (11.334%); 7139/138402 objects misplaced (5.158%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:23:06.203 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:06.205 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:23:06.238 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:06.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:05 smithi045 ceph-mon[95976]: pgmap v1073: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 15686/138402 objects degraded (11.334%); 7139/138402 objects misplaced (5.158%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:23:06.340 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:06.376 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:06.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:06.478 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:06.521 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:06.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:06.623 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:06.657 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:06.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:06.759 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:06.793 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:06.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:06.895 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:06.913 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:23:06.913 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:06 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 15686/138402 objects degraded (11.334%), 14 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:06.913 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:06 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 15686/138402 objects degraded (11.334%), 14 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:06.950 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:07.052 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:07.087 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:07.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:07.189 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:07.232 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:07.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:06 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 15686/138402 objects degraded (11.334%), 14 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:07.334 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:07.380 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:07.482 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:07.519 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:07.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:07.620 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.662 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:07.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:07.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:07.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:07.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:07.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:07.932 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:08.034 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:08.071 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:08.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:08.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.197 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:08.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[105748]: pgmap v1074: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 15259/132588 objects degraded (11.509%); 6711/132588 objects misplaced (5.062%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:23:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4013675797' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:23:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[105748]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T10:23:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[101299]: pgmap v1074: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 15259/132588 objects degraded (11.509%); 6711/132588 objects misplaced (5.062%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:23:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4013675797' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:08.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:23:08.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:08.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:07 smithi032 ceph-mon[101299]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T10:23:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:07 smithi045 ceph-mon[95976]: pgmap v1074: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 15259/132588 objects degraded (11.509%); 6711/132588 objects misplaced (5.062%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:23:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:07 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4013675797' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:07 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:23:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:07 smithi045 ceph-mon[95976]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T10:23:08.298 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:08.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:08.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:08.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:08.464 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:08.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:08.566 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:08.602 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:08.704 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:08.744 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:08.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:08.846 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:08.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:08.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:08.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:09.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.042 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:09.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:09.143 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:09.181 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:09.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:09.283 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:09.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:09.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:09.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:09.477 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:09.579 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:09.617 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:09.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:23:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:23:09] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-21T10:23:09.719 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:09.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:09.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:09.853 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:09.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.888 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:09.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:09.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:09.990 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:23:09.990 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:23:10.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:10.140 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:10.181 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:10.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:09 smithi032 ceph-mon[105748]: pgmap v1076: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 15259/131289 objects degraded (11.622%); 6711/131289 objects misplaced (5.112%); 7.7 MiB/s, 122 objects/s recovering 2024-09-21T10:23:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:09 smithi032 ceph-mon[101299]: pgmap v1076: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 15259/131289 objects degraded (11.622%); 6711/131289 objects misplaced (5.112%); 7.7 MiB/s, 122 objects/s recovering 2024-09-21T10:23:10.282 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:09 smithi045 ceph-mon[95976]: pgmap v1076: 99 pgs: 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 15259/131289 objects degraded (11.622%); 6711/131289 objects misplaced (5.112%); 7.7 MiB/s, 122 objects/s recovering 2024-09-21T10:23:10.316 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:10.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:10.451 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:10.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:10.591 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:10.692 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:10.735 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:10.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:10.807 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:23:10.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:10.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:10.990 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:11.022 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:11.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:11.124 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:11.158 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:11.163 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:11.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:11.295 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:11.397 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:11.437 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:11.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:11.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:11.566 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:11.594 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:11.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:11.696 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:11.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:11.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:11.851 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:11.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:23:11.903 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:11 smithi032 ceph-mon[101299]: pgmap v1077: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 14821/120348 objects degraded (12.315%); 6036/120348 objects misplaced (5.015%); 6.8 MiB/s, 121 objects/s recovering 2024-09-21T10:23:11.903 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:11 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:11.904 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:11 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:11.904 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:11 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14821/120348 objects degraded (12.315%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:11.904 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:11 smithi032 ceph-mon[105748]: pgmap v1077: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 14821/120348 objects degraded (12.315%); 6036/120348 objects misplaced (5.015%); 6.8 MiB/s, 121 objects/s recovering 2024-09-21T10:23:11.904 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:11 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:11.904 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:11 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:11.904 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:11 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14821/120348 objects degraded (12.315%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:11.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:11.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:12.007 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:12.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:12.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:12.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:12.184 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:12.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:11 smithi045 ceph-mon[95976]: pgmap v1077: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 14821/120348 objects degraded (12.315%); 6036/120348 objects misplaced (5.015%); 6.8 MiB/s, 121 objects/s recovering 2024-09-21T10:23:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:11 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:11 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:11 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14821/120348 objects degraded (12.315%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:12.285 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:12.315 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:12.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:12.417 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:12.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.472 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:12.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:12.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:12.645 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:12.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:12.747 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:12.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:12.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:12.878 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:12.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:12.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:23:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:23:12.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:12.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:23:12.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:23:12.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:13.025 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:13.060 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:13.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:13.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:13.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:13.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:13.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:23:13.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:23:13.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:13.302 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:13.342 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:13.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:13.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:13.482 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:13.584 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:13.631 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:13.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:13.733 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:13.737 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:13.770 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:13.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:13.872 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:13.913 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:13.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:14.015 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.056 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:14.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:14.143 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:13 smithi032 ceph-mon[101299]: pgmap v1078: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 14821/118662 objects degraded (12.490%); 6036/118662 objects misplaced (5.087%); 6.8 MiB/s, 121 objects/s recovering 2024-09-21T10:23:14.143 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:13 smithi032 ceph-mon[105748]: pgmap v1078: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 14821/118662 objects degraded (12.490%); 6036/118662 objects misplaced (5.087%); 6.8 MiB/s, 121 objects/s recovering 2024-09-21T10:23:14.157 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:14.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:14.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:13 smithi045 ceph-mon[95976]: pgmap v1078: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 14821/118662 objects degraded (12.490%); 6036/118662 objects misplaced (5.087%); 6.8 MiB/s, 121 objects/s recovering 2024-09-21T10:23:14.297 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:14.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:14.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:14.472 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:14.571 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:23:14.571 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:23:14.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:14.611 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:23:14.611 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:23:14.612 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:14.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:14.650 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:23:14.650 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:23:14.685 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:23:14.685 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:23:14.714 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:14.731 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:23:14.731 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:23:14.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:14.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:14.770 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:23:14.770 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:23:14.802 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:23:14.802 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:23:14.835 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:23:14.836 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:23:14.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:14.897 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:14.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:14 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3422576735' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:14.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:14 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3422576735' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:14.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:15.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:15.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:15.142 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:15.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.179 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:15.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:15.280 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:15.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:14 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3422576735' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:15.315 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:15.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:15.417 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:15.452 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:15.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:15.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:15.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:15.747 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:15.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:15.848 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:15.883 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:15.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:15.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:16.016 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:16.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:16.118 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:16.158 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:16.169 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:15 smithi032 ceph-mon[101299]: pgmap v1079: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 14821/114573 objects degraded (12.936%); 5490/114573 objects misplaced (4.792%); 5.4 MiB/s, 100 objects/s recovering 2024-09-21T10:23:16.169 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:15 smithi032 ceph-mon[105748]: pgmap v1079: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 14821/114573 objects degraded (12.936%); 5490/114573 objects misplaced (4.792%); 5.4 MiB/s, 100 objects/s recovering 2024-09-21T10:23:16.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:15 smithi045 ceph-mon[95976]: pgmap v1079: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 14821/114573 objects degraded (12.936%); 5490/114573 objects misplaced (4.792%); 5.4 MiB/s, 100 objects/s recovering 2024-09-21T10:23:16.300 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:16.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:16.402 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:16.435 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:16.502 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:16.537 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:16.571 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:16.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:16.673 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:16.708 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:16.810 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:16.851 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:16.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:16.953 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:16.983 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:16.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:17.085 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:17.127 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:17.156 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:23:17.158 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:23:17.175 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:16 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14821/114573 objects degraded (12.936%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:17.175 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:16 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14821/114573 objects degraded (12.936%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:17.186 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:16 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14821/114573 objects degraded (12.936%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:17.230 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:17.269 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:17.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:17.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:17.406 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:17.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:17.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.548 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:17.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:17.649 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:17.689 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:17.791 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:17.834 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:17.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:17.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:17.969 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:17.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:17.983 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:23:17.983 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:23:17.983 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.983 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:167128 8.c 8:3ca06cd1:::benchmark_data_smithi032_233595_object67283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-21T10:23:17.983 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:17.960297+0000", 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.020066736000000002, 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.020089072999999999, 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 167128 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:23:17.984 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960297+0000", 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960297+0000", 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960298+0000", 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.032e-06 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960302+0000", 2024-09-21T10:23:17.985 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5040000000000004e-06 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960313+0000", 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0747999999999999e-05 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960320+0000", 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9349999999999997e-06 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:23:17.986 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.964751+0000", 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0044313620000000003 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.964871+0000", 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011972999999999999 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4,5,7", 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.965004+0000", 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000133094 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:17.987 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.972821+0000", 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0078171109999999999 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.972844+0000", 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2832000000000001e-05 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.973751+0000", 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00090740099999999995 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.988 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978999+0000", 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0052480529999999999 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:167129 8.8 8:17db840c:::benchmark_data_smithi032_233595_object67284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:17.960345+0000", 2024-09-21T10:23:17.989 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.020018755999999999, 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.02009559, 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 167129 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960345+0000", 2024-09-21T10:23:17.990 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960345+0000", 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960345+0000", 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9100000000000004e-07 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.991 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960348+0000", 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.694e-06 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960354+0000", 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2260000000000002e-06 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960359+0000", 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5719999999999996e-06 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.992 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960653+0000", 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029403600000000002 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960781+0000", 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012859300000000001 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 5,7", 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960909+0000", 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012767599999999999 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.993 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.968647+0000", 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00773787 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.968664+0000", 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7175000000000001e-05 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.994 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:167130 8.8 8:12a61564:::benchmark_data_smithi032_233595_object67285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:17.960491+0000", 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.019871832999999998, 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.019989036000000002, 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 167130 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:23:17.995 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960491+0000", 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960491+0000", 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960492+0000", 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5400000000000001e-07 2024-09-21T10:23:17.996 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.997 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.997 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:23:17.997 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960496+0000", 2024-09-21T10:23:17.997 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4020000000000002e-06 2024-09-21T10:23:17.997 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.997 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.997 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960505+0000", 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2099999999999999e-06 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.960511+0000", 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6699999999999997e-06 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:23:17.998 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.964680+0000", 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0041688719999999997 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.964824+0000", 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014328799999999999 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 5,7", 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.964979+0000", 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015541900000000001 2024-09-21T10:23:17.999 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.968672+0000", 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0036934910000000001 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.970041+0000", 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0013680789999999999 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.000 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:167133 8.0 8:0d18fb1d:::benchmark_data_smithi032_233595_object67288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:17.968738+0000", 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.011625313, 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011783353, 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 167133 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.001 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.968738+0000", 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.968738+0000", 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.968740+0000", 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7290000000000001e-06 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.002 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.003 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:23:18.003 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.968744+0000", 2024-09-21T10:23:18.003 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.369e-06 2024-09-21T10:23:18.003 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.003 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.003 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:23:18.003 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.968757+0000", 2024-09-21T10:23:18.003 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2734999999999999e-05 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.968764+0000", 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1169999999999998e-06 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.969209+0000", 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000445518 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.004 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.969352+0000", 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014236000000000001 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.969493+0000", 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000140844 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.973288+0000", 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0037956410000000002 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.005 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975246+0000", 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0019580940000000001 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:167118 8.c 8:324d6f61:::benchmark_data_smithi032_233595_object67273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:17.942745+0000", 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.037617838000000001, 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.037815070999999999, 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:23:18.006 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 167118 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.942745+0000", 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.007 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.942745+0000", 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.942746+0000", 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8400000000000004e-07 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.942749+0000", 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.416e-06 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.008 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.942770+0000", 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0971999999999998e-05 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.942777+0000", 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8499999999999996e-06 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.942862+0000", 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4810999999999998e-05 2024-09-21T10:23:18.009 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.943047+0000", 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018517699999999999 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4,5,7", 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.943242+0000", 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00019502199999999999 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:18.010 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.947549+0000", 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0043071029999999996 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.947568+0000", 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8479999999999999e-05 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.948880+0000", 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001311965 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:18.011 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.950607+0000", 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0017273119999999999 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:167138 8.b 8:db21bd32:::benchmark_data_smithi032_233595_object67293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:17.975059+0000", 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0053043839999999997, 2024-09-21T10:23:18.012 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0055461870000000002, 2024-09-21T10:23:18.013 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:23:18.013 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:23:18.013 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:23:18.013 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:23:18.013 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:23:18.013 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:18.013 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 167138 2024-09-21T10:23:18.014 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.014 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:23:18.014 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.014 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:23:18.014 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975059+0000", 2024-09-21T10:23:18.014 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.014 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.015 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.015 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:23:18.015 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975059+0000", 2024-09-21T10:23:18.015 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.015 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.015 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.016 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:23:18.016 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975060+0000", 2024-09-21T10:23:18.016 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.595e-06 2024-09-21T10:23:18.016 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.016 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.016 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:23:18.017 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975066+0000", 2024-09-21T10:23:18.017 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3210000000000004e-06 2024-09-21T10:23:18.017 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.017 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.017 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:23:18.017 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975080+0000", 2024-09-21T10:23:18.018 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4453999999999999e-05 2024-09-21T10:23:18.018 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.018 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.018 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:23:18.018 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975088+0000", 2024-09-21T10:23:18.018 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0360000000000006e-06 2024-09-21T10:23:18.019 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.019 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.019 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:23:18.019 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975172+0000", 2024-09-21T10:23:18.019 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3405000000000005e-05 2024-09-21T10:23:18.019 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.019 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.020 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:23:18.020 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975315+0000", 2024-09-21T10:23:18.020 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000143091 2024-09-21T10:23:18.020 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.020 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.020 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:23:18.020 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.975433+0000", 2024-09-21T10:23:18.021 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000118492 2024-09-21T10:23:18.021 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.021 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.021 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:23:18.021 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978919+0000", 2024-09-21T10:23:18.021 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0034857569999999999 2024-09-21T10:23:18.021 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.022 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:23:18.022 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.022 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.022 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.022 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:167143 8.f 8:f01716af:::benchmark_data_smithi032_233595_object67298:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-21T10:23:18.022 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:17.978784+0000", 2024-09-21T10:23:18.023 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0015789949999999999, 2024-09-21T10:23:18.023 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0018559430000000001, 2024-09-21T10:23:18.023 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:23:18.023 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:23:18.023 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:23:18.023 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:23:18.024 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:23:18.024 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:18.024 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 167143 2024-09-21T10:23:18.024 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.024 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:23:18.024 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.025 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:23:18.025 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978784+0000", 2024-09-21T10:23:18.025 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.025 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.025 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.025 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:23:18.025 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978784+0000", 2024-09-21T10:23:18.026 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.026 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.026 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.026 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:23:18.026 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978786+0000", 2024-09-21T10:23:18.026 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.674e-06 2024-09-21T10:23:18.026 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.027 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.027 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:23:18.027 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978790+0000", 2024-09-21T10:23:18.027 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3699999999999997e-06 2024-09-21T10:23:18.027 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.027 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.028 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:23:18.028 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978802+0000", 2024-09-21T10:23:18.028 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1873e-05 2024-09-21T10:23:18.028 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.028 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.028 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:23:18.028 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978809+0000", 2024-09-21T10:23:18.029 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5180000000000001e-06 2024-09-21T10:23:18.029 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.029 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.029 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:23:18.029 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978857+0000", 2024-09-21T10:23:18.029 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8040000000000001e-05 2024-09-21T10:23:18.029 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.030 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.030 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:23:18.030 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.978982+0000", 2024-09-21T10:23:18.030 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012559600000000001 2024-09-21T10:23:18.030 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.031 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.031 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:23:18.031 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.979081+0000", 2024-09-21T10:23:18.031 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8963999999999996e-05 2024-09-21T10:23:18.031 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.031 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:23:18.031 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.032 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.032 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.032 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.35586.0:167126 8.c 8:3f31ee0d:::benchmark_data_smithi032_233595_object67281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-21T10:23:18.032 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:17.955919+0000", 2024-09-21T10:23:18.032 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.024444761999999998, 2024-09-21T10:23:18.032 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.024754766000000001, 2024-09-21T10:23:18.032 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:23:18.033 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:23:18.033 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:23:18.033 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:23:18.033 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.35586", 2024-09-21T10:23:18.033 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:18.033 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 167126 2024-09-21T10:23:18.034 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.034 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:23:18.034 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.034 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:23:18.034 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.955919+0000", 2024-09-21T10:23:18.034 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.034 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.035 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.035 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:23:18.035 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.955919+0000", 2024-09-21T10:23:18.035 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:23:18.035 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.036 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.036 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:23:18.036 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.955920+0000", 2024-09-21T10:23:18.036 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5069999999999999e-06 2024-09-21T10:23:18.036 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.036 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.036 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:23:18.036 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.955925+0000", 2024-09-21T10:23:18.037 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0100000000000003e-06 2024-09-21T10:23:18.037 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.037 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.037 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:23:18.037 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.955940+0000", 2024-09-21T10:23:18.037 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4511e-05 2024-09-21T10:23:18.037 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.955949+0000", 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0259999999999996e-06 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.956061+0000", 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000112222 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.038 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.956226+0000", 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000165489 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4,5,7", 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.956381+0000", 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000154518 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.969769+0000", 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013388035 2024-09-21T10:23:18.039 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.970364+0000", 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00059554900000000001 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.972838+0000", 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0024738659999999999 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:17.977591+0000", 2024-09-21T10:23:18.040 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0047529540000000002 2024-09-21T10:23:18.041 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.041 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:23:18.041 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.041 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:23:18.041 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:23:18.041 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 8 2024-09-21T10:23:18.041 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:23:18.071 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:18.099 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:18.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:18.167 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:17 smithi045 ceph-mon[95976]: pgmap v1080: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 14821/107319 objects degraded (13.810%); 5223/107319 objects misplaced (4.867%); 2.8 MiB/s, 95 objects/s recovering 2024-09-21T10:23:18.167 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:17 smithi045 ceph-mon[95976]: from='client.45743 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:23:18.202 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:18.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:17 smithi032 ceph-mon[101299]: pgmap v1080: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 14821/107319 objects degraded (13.810%); 5223/107319 objects misplaced (4.867%); 2.8 MiB/s, 95 objects/s recovering 2024-09-21T10:23:18.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:17 smithi032 ceph-mon[101299]: from='client.45743 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:23:18.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:17 smithi032 ceph-mon[105748]: pgmap v1080: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 14821/107319 objects degraded (13.810%); 5223/107319 objects misplaced (4.867%); 2.8 MiB/s, 95 objects/s recovering 2024-09-21T10:23:18.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:17 smithi032 ceph-mon[105748]: from='client.45743 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:23:18.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:18.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:18.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:18.371 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:18.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:18.473 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:18.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:18.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:18.586 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1080,"stamp":"2024-09-21T10:23:16.536569+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2347309053,"num_objects":35773,"num_object_clones":0,"num_object_copies":107319,"num_objects_missing_on_primary":939,"num_objects_missing":939,"num_objects_degraded":14821,"num_objects_misplaced":5223,"num_objects_unfound":0,"num_objects_dirty":35773,"num_whiteouts":0,"num_read":6389,"num_read_kb":13844,"num_write":165064,"num_write_kb":6404930,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23109,"num_bytes_recovered":1352433657,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110585,"ondisk_log_size":110585,"up":297,"acting":289,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12260732,"kb_used_data":7668412,"kb_used_omap":40,"kb_used_meta":4591831,"kb_avail":737700484,"statfs":{"total":767960285184,"available":755405295616,"internally_reserved":0,"allocated":7852453888,"data_stored":7837774117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41824,"internal_metadata":4702035104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-552009728,"num_objects":-8423,"num_object_clones":0,"num_object_copies":-25269,"num_objects_missing_on_primary":-639,"num_objects_missing":-639,"num_objects_degraded":-438,"num_objects_misplaced":-1488,"num_objects_unfound":0,"num_objects_dirty":-8423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8423,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":957,"num_bytes_recovered":29097984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003639"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":593,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.909350+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:23:06.909350+0000","last_peered":"2024-09-21T10:23:06.909350+0000","last_clean":"2024-09-21T10:23:06.909350+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:23:06.909350+0000","last_undegraded":"2024-09-21T10:23:06.909350+0000","last_fullsized":"2024-09-21T10:23:06.909350+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":361,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:23:04.877835+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:23:04.877835+0000","last_peered":"2024-09-21T10:23:04.877835+0000","last_clean":"2024-09-21T10:23:04.877835+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:23:04.877835+0000","last_undegraded":"2024-09-21T10:23:04.877835+0000","last_fullsized":"2024-09-21T10:23:04.877835+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.636303+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:22:49.636303+0000","last_peered":"2024-09-21T10:22:49.636303+0000","last_clean":"2024-09-21T10:22:49.636303+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:22:49.636303+0000","last_undegraded":"2024-09-21T10:22:49.636303+0000","last_fullsized":"2024-09-21T10:22:49.636303+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":34,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.921315+0000","last_change":"2024-09-21T10:22:01.318638+0000","last_active":"2024-09-21T10:23:06.921315+0000","last_peered":"2024-09-21T10:23:06.921315+0000","last_clean":"2024-09-21T10:23:06.921315+0000","last_became_active":"2024-09-21T10:22:01.317087+0000","last_became_peered":"2024-09-21T10:22:01.317087+0000","last_unstale":"2024-09-21T10:23:06.921315+0000","last_undegraded":"2024-09-21T10:23:06.921315+0000","last_fullsized":"2024-09-21T10:23:06.921315+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:25.334726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":35,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.921595+0000","last_change":"2024-09-21T10:22:01.318978+0000","last_active":"2024-09-21T10:23:06.921595+0000","last_peered":"2024-09-21T10:23:06.921595+0000","last_clean":"2024-09-21T10:23:06.921595+0000","last_became_active":"2024-09-21T10:22:01.318255+0000","last_became_peered":"2024-09-21T10:22:01.318255+0000","last_unstale":"2024-09-21T10:23:06.921595+0000","last_undegraded":"2024-09-21T10:23:06.921595+0000","last_fullsized":"2024-09-21T10:23:06.921595+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:55.084088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":332,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:23:04.877781+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:23:04.877781+0000","last_peered":"2024-09-21T10:23:04.877781+0000","last_clean":"2024-09-21T10:23:04.877781+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:23:04.877781+0000","last_undegraded":"2024-09-21T10:23:04.877781+0000","last_fullsized":"2024-09-21T10:23:04.877781+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":614,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.909080+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:23:06.909080+0000","last_peered":"2024-09-21T10:23:06.909080+0000","last_clean":"2024-09-21T10:23:06.909080+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:23:06.909080+0000","last_undegraded":"2024-09-21T10:23:06.909080+0000","last_fullsized":"2024-09-21T10:23:06.909080+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":413,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:59.953807+0000","last_change":"2024-09-21T10:22:01.331612+0000","last_active":"2024-09-21T10:22:59.953807+0000","last_peered":"2024-09-21T10:22:59.953807+0000","last_clean":"2024-09-21T10:22:59.953807+0000","last_became_active":"2024-09-21T10:22:01.325282+0000","last_became_peered":"2024-09-21T10:22:01.325282+0000","last_unstale":"2024-09-21T10:22:59.953807+0000","last_undegraded":"2024-09-21T10:22:59.953807+0000","last_fullsized":"2024-09-21T10:22:59.953807+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:23.383106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"358'10302","reported_seq":11367,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:15.015345+0000","last_change":"2024-09-21T10:21:39.510930+0000","last_active":"2024-09-21T10:23:15.015345+0000","last_peered":"2024-09-21T10:23:15.015345+0000","last_clean":"2024-09-21T10:20:12.056605+0000","last_became_active":"2024-09-21T10:20:14.466147+0000","last_became_peered":"2024-09-21T10:20:14.466147+0000","last_unstale":"2024-09-21T10:23:15.015345+0000","last_undegraded":"2024-09-21T10:20:14.094800+0000","last_fullsized":"2024-09-21T10:20:14.071992+0000","mapping_epoch":325,"log_start":"319'2900","ondisk_log_start":"319'2900","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7402,"log_dups_size":0,"ondisk_log_size":7402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:34.237367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145358848,"num_objects":2218,"num_object_clones":0,"num_object_copies":6654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3167,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10302,"num_write_kb":400640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":489,"num_bytes_recovered":32047104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-949}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":47,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611735+0000","last_change":"2024-09-21T10:22:01.321467+0000","last_active":"2024-09-21T10:22:49.611735+0000","last_peered":"2024-09-21T10:22:49.611735+0000","last_clean":"2024-09-21T10:22:49.611735+0000","last_became_active":"2024-09-21T10:22:01.318270+0000","last_became_peered":"2024-09-21T10:22:01.318270+0000","last_unstale":"2024-09-21T10:22:49.611735+0000","last_undegraded":"2024-09-21T10:22:49.611735+0000","last_fullsized":"2024-09-21T10:22:49.611735+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:44.059553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":65,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606452+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:22:49.606452+0000","last_peered":"2024-09-21T10:22:49.606452+0000","last_clean":"2024-09-21T10:22:49.606452+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:22:49.606452+0000","last_undegraded":"2024-09-21T10:22:49.606452+0000","last_fullsized":"2024-09-21T10:22:49.606452+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"358'10086","reported_seq":13959,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:14.952630+0000","last_change":"2024-09-21T10:20:04.684629+0000","last_active":"2024-09-21T10:23:14.952630+0000","last_peered":"2024-09-21T10:23:14.952630+0000","last_clean":"2024-09-21T10:23:14.952630+0000","last_became_active":"2024-09-21T10:20:04.680299+0000","last_became_peered":"2024-09-21T10:20:04.680299+0000","last_unstale":"2024-09-21T10:23:14.952630+0000","last_undegraded":"2024-09-21T10:23:14.952630+0000","last_fullsized":"2024-09-21T10:23:14.952630+0000","mapping_epoch":320,"log_start":"323'3500","ondisk_log_start":"323'3500","created":305,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6586,"log_dups_size":0,"ondisk_log_size":6586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:41.743081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141819904,"num_objects":2164,"num_object_clones":0,"num_object_copies":6492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10086,"num_write_kb":392000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1891,"num_bytes_recovered":123928576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":34,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.612468+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:22:49.612468+0000","last_peered":"2024-09-21T10:22:49.612468+0000","last_clean":"2024-09-21T10:22:49.612468+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:22:49.612468+0000","last_undegraded":"2024-09-21T10:22:49.612468+0000","last_fullsized":"2024-09-21T10:22:49.612468+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":54,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606193+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:22:49.606193+0000","last_peered":"2024-09-21T10:22:49.606193+0000","last_clean":"2024-09-21T10:22:49.606193+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:22:49.606193+0000","last_undegraded":"2024-09-21T10:22:49.606193+0000","last_fullsized":"2024-09-21T10:22:49.606193+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"358'9897","reported_seq":10015,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:11.629771+0000","last_change":"2024-09-21T10:19:22.649712+0000","last_active":"2024-09-21T10:23:11.629771+0000","last_peered":"2024-09-21T10:23:11.629771+0000","last_clean":"2024-09-21T10:23:11.629771+0000","last_became_active":"2024-09-21T10:19:22.649303+0000","last_became_peered":"2024-09-21T10:19:22.649303+0000","last_unstale":"2024-09-21T10:23:11.629771+0000","last_undegraded":"2024-09-21T10:23:11.629771+0000","last_fullsized":"2024-09-21T10:23:11.629771+0000","mapping_epoch":305,"log_start":"319'2661","ondisk_log_start":"319'2661","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7236,"log_dups_size":0,"ondisk_log_size":7236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:24.442851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157745152,"num_objects":2407,"num_object_clones":0,"num_object_copies":7221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9897,"num_write_kb":393728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":356,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.613140+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:22:49.613140+0000","last_peered":"2024-09-21T10:22:49.613140+0000","last_clean":"2024-09-21T10:22:49.613140+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:22:49.613140+0000","last_undegraded":"2024-09-21T10:22:49.613140+0000","last_fullsized":"2024-09-21T10:22:49.613140+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":376,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:59.953848+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:22:59.953848+0000","last_peered":"2024-09-21T10:22:59.953848+0000","last_clean":"2024-09-21T10:22:59.953848+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:22:59.953848+0000","last_undegraded":"2024-09-21T10:22:59.953848+0000","last_fullsized":"2024-09-21T10:22:59.953848+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"358'10614","reported_seq":10732,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:15.404512+0000","last_change":"2024-09-21T10:19:22.640645+0000","last_active":"2024-09-21T10:23:15.404512+0000","last_peered":"2024-09-21T10:23:15.404512+0000","last_clean":"2024-09-21T10:23:15.404512+0000","last_became_active":"2024-09-21T10:19:22.640343+0000","last_became_peered":"2024-09-21T10:19:22.640343+0000","last_unstale":"2024-09-21T10:23:15.404512+0000","last_undegraded":"2024-09-21T10:23:15.404512+0000","last_fullsized":"2024-09-21T10:23:15.404512+0000","mapping_epoch":305,"log_start":"331'5300","ondisk_log_start":"331'5300","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5314,"log_dups_size":0,"ondisk_log_size":5314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:48.734122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146014208,"num_objects":2228,"num_object_clones":0,"num_object_copies":6684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10614,"num_write_kb":410944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":290,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606645+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:22:49.606645+0000","last_peered":"2024-09-21T10:22:49.606645+0000","last_clean":"2024-09-21T10:22:49.606645+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:22:49.606645+0000","last_undegraded":"2024-09-21T10:22:49.606645+0000","last_fullsized":"2024-09-21T10:22:49.606645+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":204,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.921379+0000","last_change":"2024-09-21T10:19:50.585724+0000","last_active":"2024-09-21T10:23:06.921379+0000","last_peered":"2024-09-21T10:23:06.921379+0000","last_clean":"2024-09-21T10:23:06.921379+0000","last_became_active":"2024-09-21T10:19:50.585315+0000","last_became_peered":"2024-09-21T10:19:50.585315+0000","last_unstale":"2024-09-21T10:23:06.921379+0000","last_undegraded":"2024-09-21T10:23:06.921379+0000","last_fullsized":"2024-09-21T10:23:06.921379+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:26.618970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"358'10265","reported_seq":15346,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:13.810918+0000","last_change":"2024-09-21T10:22:01.328242+0000","last_active":"2024-09-21T10:23:13.810918+0000","last_peered":"2024-09-21T10:23:13.810918+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492359+0000","last_became_peered":"2024-09-21T10:20:14.492359+0000","last_unstale":"2024-09-21T10:23:13.810918+0000","last_undegraded":"2024-09-21T10:20:14.218670+0000","last_fullsized":"2024-09-21T10:20:14.071510+0000","mapping_epoch":325,"log_start":"322'3400","ondisk_log_start":"322'3400","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6865,"log_dups_size":0,"ondisk_log_size":6865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148045824,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10265,"num_write_kb":400768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2481,"num_bytes_recovered":148307968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":1055}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611439+0000","last_change":"2024-09-21T10:22:04.029688+0000","last_active":"2024-09-21T10:22:49.611439+0000","last_peered":"2024-09-21T10:22:49.611439+0000","last_clean":"2024-09-21T10:22:49.611439+0000","last_became_active":"2024-09-21T10:22:04.028853+0000","last_became_peered":"2024-09-21T10:22:04.028853+0000","last_unstale":"2024-09-21T10:22:49.611439+0000","last_undegraded":"2024-09-21T10:22:49.611439+0000","last_fullsized":"2024-09-21T10:22:49.611439+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:20.457596+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:19:20.457596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":643,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.908660+0000","last_change":"2024-09-21T10:22:01.335303+0000","last_active":"2024-09-21T10:23:06.908660+0000","last_peered":"2024-09-21T10:23:06.908660+0000","last_clean":"2024-09-21T10:23:06.908660+0000","last_became_active":"2024-09-21T10:22:01.325298+0000","last_became_peered":"2024-09-21T10:22:01.325298+0000","last_unstale":"2024-09-21T10:23:06.908660+0000","last_undegraded":"2024-09-21T10:23:06.908660+0000","last_fullsized":"2024-09-21T10:23:06.908660+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:01.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":54,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606774+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:22:49.606774+0000","last_peered":"2024-09-21T10:22:49.606774+0000","last_clean":"2024-09-21T10:22:49.606774+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:22:49.606774+0000","last_undegraded":"2024-09-21T10:22:49.606774+0000","last_fullsized":"2024-09-21T10:22:49.606774+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"358'10139","reported_seq":10257,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:15.406686+0000","last_change":"2024-09-21T10:19:22.641281+0000","last_active":"2024-09-21T10:23:15.406686+0000","last_peered":"2024-09-21T10:23:15.406686+0000","last_clean":"2024-09-21T10:23:15.406686+0000","last_became_active":"2024-09-21T10:19:22.640988+0000","last_became_peered":"2024-09-21T10:19:22.640988+0000","last_unstale":"2024-09-21T10:23:15.406686+0000","last_undegraded":"2024-09-21T10:23:15.406686+0000","last_fullsized":"2024-09-21T10:23:15.406686+0000","mapping_epoch":305,"log_start":"328'4800","ondisk_log_start":"328'4800","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5339,"log_dups_size":0,"ondisk_log_size":5339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:52.652565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135200768,"num_objects":2063,"num_object_clones":0,"num_object_copies":6189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10139,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":426,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:49.613170+0000","last_change":"2024-09-21T10:22:02.013601+0000","last_active":"2024-09-21T10:22:49.613170+0000","last_peered":"2024-09-21T10:22:49.613170+0000","last_clean":"2024-09-21T10:19:15.364615+0000","last_became_active":"2024-09-21T10:22:02.013140+0000","last_became_peered":"2024-09-21T10:22:02.013140+0000","last_unstale":"2024-09-21T10:22:49.613170+0000","last_undegraded":"2024-09-21T10:22:02.004849+0000","last_fullsized":"2024-09-21T10:22:02.002552+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_clean_scrub_stamp":"2024-09-21T10:19:15.365774+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1},{"shards":"6,7","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":366,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:59.953865+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:22:59.953865+0000","last_peered":"2024-09-21T10:22:59.953865+0000","last_clean":"2024-09-21T10:22:59.953865+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:22:59.953865+0000","last_undegraded":"2024-09-21T10:22:59.953865+0000","last_fullsized":"2024-09-21T10:22:59.953865+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":54,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.607353+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:22:49.607353+0000","last_peered":"2024-09-21T10:22:49.607353+0000","last_clean":"2024-09-21T10:22:49.607353+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:22:49.607353+0000","last_undegraded":"2024-09-21T10:22:49.607353+0000","last_fullsized":"2024-09-21T10:22:49.607353+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"358'10374","reported_seq":17703,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:15.031896+0000","last_change":"2024-09-21T10:22:34.604937+0000","last_active":"2024-09-21T10:23:15.031896+0000","last_peered":"2024-09-21T10:23:15.031896+0000","last_clean":"2024-09-21T10:23:15.031896+0000","last_became_active":"2024-09-21T10:22:34.577215+0000","last_became_peered":"2024-09-21T10:22:34.577215+0000","last_unstale":"2024-09-21T10:23:15.031896+0000","last_undegraded":"2024-09-21T10:23:15.031896+0000","last_fullsized":"2024-09-21T10:23:15.031896+0000","mapping_epoch":354,"log_start":"319'3000","ondisk_log_start":"319'3000","created":305,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:40.287923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146800640,"num_objects":2240,"num_object_clones":0,"num_object_copies":6720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10374,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3618,"num_bytes_recovered":121634816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":54,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606003+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:22:49.606003+0000","last_peered":"2024-09-21T10:22:49.606003+0000","last_clean":"2024-09-21T10:22:49.606003+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:22:49.606003+0000","last_undegraded":"2024-09-21T10:22:49.606003+0000","last_fullsized":"2024-09-21T10:22:49.606003+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":54,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.607600+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:22:49.607600+0000","last_peered":"2024-09-21T10:22:49.607600+0000","last_clean":"2024-09-21T10:22:49.607600+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:22:49.607600+0000","last_undegraded":"2024-09-21T10:22:49.607600+0000","last_fullsized":"2024-09-21T10:22:49.607600+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"358'10309","reported_seq":10440,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:15.402484+0000","last_change":"2024-09-21T10:21:39.558048+0000","last_active":"2024-09-21T10:23:15.402484+0000","last_peered":"2024-09-21T10:23:15.402484+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492538+0000","last_became_peered":"2024-09-21T10:20:14.492538+0000","last_unstale":"2024-09-21T10:23:15.402484+0000","last_undegraded":"2024-09-21T10:20:14.216051+0000","last_fullsized":"2024-09-21T10:20:14.075057+0000","mapping_epoch":325,"log_start":"329'5000","ondisk_log_start":"329'5000","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5309,"log_dups_size":0,"ondisk_log_size":5309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145031168,"num_objects":2213,"num_object_clones":0,"num_object_copies":6639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10309,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1363}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611675+0000","last_change":"2024-09-21T10:22:04.031320+0000","last_active":"2024-09-21T10:22:49.611675+0000","last_peered":"2024-09-21T10:22:49.611675+0000","last_clean":"2024-09-21T10:22:49.611675+0000","last_became_active":"2024-09-21T10:22:04.030774+0000","last_became_peered":"2024-09-21T10:22:04.030774+0000","last_unstale":"2024-09-21T10:22:49.611675+0000","last_undegraded":"2024-09-21T10:22:49.611675+0000","last_fullsized":"2024-09-21T10:22:49.611675+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:22.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:22.489745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":621,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.909962+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:23:06.909962+0000","last_peered":"2024-09-21T10:23:06.909962+0000","last_clean":"2024-09-21T10:23:06.909962+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:23:06.909962+0000","last_undegraded":"2024-09-21T10:23:06.909962+0000","last_fullsized":"2024-09-21T10:23:06.909962+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":314,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.918866+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:23:06.918866+0000","last_peered":"2024-09-21T10:23:06.918866+0000","last_clean":"2024-09-21T10:23:06.918866+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:23:06.918866+0000","last_undegraded":"2024-09-21T10:23:06.918866+0000","last_fullsized":"2024-09-21T10:23:06.918866+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"358'10294","reported_seq":14131,"reported_epoch":358,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:23:14.941705+0000","last_change":"2024-09-21T10:22:02.106012+0000","last_active":"2024-09-21T10:23:14.941705+0000","last_peered":"2024-09-21T10:23:14.941705+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:22:02.086281+0000","last_became_peered":"2024-09-21T10:22:02.086281+0000","last_unstale":"2024-09-21T10:23:14.941705+0000","last_undegraded":"2024-09-21T10:22:02.081791+0000","last_fullsized":"2024-09-21T10:23:14.941705+0000","mapping_epoch":347,"log_start":"326'3700","ondisk_log_start":"326'3700","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6594,"log_dups_size":0,"ondisk_log_size":6594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:37.553227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140771328,"num_objects":2148,"num_object_clones":0,"num_object_copies":6444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1793,"num_objects_misplaced":2148,"num_objects_unfound":0,"num_objects_dirty":2148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10294,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1878,"num_bytes_recovered":123076608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,5,6","objects":2148}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.613016+0000","last_change":"2024-09-21T10:22:04.029284+0000","last_active":"2024-09-21T10:22:49.613016+0000","last_peered":"2024-09-21T10:22:49.613016+0000","last_clean":"2024-09-21T10:22:49.613016+0000","last_became_active":"2024-09-21T10:22:04.028938+0000","last_became_peered":"2024-09-21T10:22:04.028938+0000","last_unstale":"2024-09-21T10:22:49.613016+0000","last_undegraded":"2024-09-21T10:22:49.613016+0000","last_fullsized":"2024-09-21T10:22:49.613016+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:12.388304+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:12.388304+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":642,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:49.608991+0000","last_change":"2024-09-21T10:21:36.010162+0000","last_active":"2024-09-21T10:22:49.608991+0000","last_peered":"2024-09-21T10:22:49.608991+0000","last_clean":"2024-09-21T10:21:17.725138+0000","last_became_active":"2024-09-21T10:21:36.009706+0000","last_became_peered":"2024-09-21T10:21:36.009706+0000","last_unstale":"2024-09-21T10:22:49.608991+0000","last_undegraded":"2024-09-21T10:21:35.852171+0000","last_fullsized":"2024-09-21T10:21:35.851526+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:04.206298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":553,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.908499+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:23:06.908499+0000","last_peered":"2024-09-21T10:23:06.908499+0000","last_clean":"2024-09-21T10:23:06.908499+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:23:06.908499+0000","last_undegraded":"2024-09-21T10:23:06.908499+0000","last_fullsized":"2024-09-21T10:23:06.908499+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":332,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.612084+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:22:49.612084+0000","last_peered":"2024-09-21T10:22:49.612084+0000","last_clean":"2024-09-21T10:22:49.612084+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:22:49.612084+0000","last_undegraded":"2024-09-21T10:22:49.612084+0000","last_fullsized":"2024-09-21T10:22:49.612084+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"358'10013","reported_seq":10131,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:15.396425+0000","last_change":"2024-09-21T10:20:14.480108+0000","last_active":"2024-09-21T10:23:15.396425+0000","last_peered":"2024-09-21T10:23:15.396425+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.471864+0000","last_became_peered":"2024-09-21T10:20:14.471864+0000","last_unstale":"2024-09-21T10:23:15.396425+0000","last_undegraded":"2024-09-21T10:20:14.214064+0000","last_fullsized":"2024-09-21T10:20:14.074889+0000","mapping_epoch":325,"log_start":"328'4700","ondisk_log_start":"328'4700","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5313,"log_dups_size":0,"ondisk_log_size":5313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143589376,"num_objects":2191,"num_object_clones":0,"num_object_copies":6573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10013,"num_write_kb":390528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-1354}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611837+0000","last_change":"2024-09-21T10:22:04.031573+0000","last_active":"2024-09-21T10:22:49.611837+0000","last_peered":"2024-09-21T10:22:49.611837+0000","last_clean":"2024-09-21T10:22:49.611837+0000","last_became_active":"2024-09-21T10:22:04.031125+0000","last_became_peered":"2024-09-21T10:22:04.031125+0000","last_unstale":"2024-09-21T10:22:49.611837+0000","last_undegraded":"2024-09-21T10:22:49.611837+0000","last_fullsized":"2024-09-21T10:22:49.611837+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:19:19.450903+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":997,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.613307+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:22:49.613307+0000","last_peered":"2024-09-21T10:22:49.613307+0000","last_clean":"2024-09-21T10:22:49.613307+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:22:49.613307+0000","last_undegraded":"2024-09-21T10:22:49.613307+0000","last_fullsized":"2024-09-21T10:22:49.613307+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":203,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.920899+0000","last_change":"2024-09-21T10:19:51.555124+0000","last_active":"2024-09-21T10:23:06.920899+0000","last_peered":"2024-09-21T10:23:06.920899+0000","last_clean":"2024-09-21T10:23:06.920899+0000","last_became_active":"2024-09-21T10:19:51.554766+0000","last_became_peered":"2024-09-21T10:19:51.554766+0000","last_unstale":"2024-09-21T10:23:06.920899+0000","last_undegraded":"2024-09-21T10:23:06.920899+0000","last_fullsized":"2024-09-21T10:23:06.920899+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:52.832950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":34,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.921543+0000","last_change":"2024-09-21T10:22:01.316140+0000","last_active":"2024-09-21T10:23:06.921543+0000","last_peered":"2024-09-21T10:23:06.921543+0000","last_clean":"2024-09-21T10:23:06.921543+0000","last_became_active":"2024-09-21T10:22:01.315408+0000","last_became_peered":"2024-09-21T10:22:01.315408+0000","last_unstale":"2024-09-21T10:23:06.921543+0000","last_undegraded":"2024-09-21T10:23:06.921543+0000","last_fullsized":"2024-09-21T10:23:06.921543+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:07.322279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"358'10251","reported_seq":14230,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:13.820636+0000","last_change":"2024-09-21T10:20:47.589372+0000","last_active":"2024-09-21T10:23:13.820636+0000","last_peered":"2024-09-21T10:23:13.820636+0000","last_clean":"2024-09-21T10:23:13.820636+0000","last_became_active":"2024-09-21T10:20:47.580051+0000","last_became_peered":"2024-09-21T10:20:47.580051+0000","last_unstale":"2024-09-21T10:23:13.820636+0000","last_undegraded":"2024-09-21T10:23:13.820636+0000","last_fullsized":"2024-09-21T10:23:13.820636+0000","mapping_epoch":332,"log_start":"322'3400","ondisk_log_start":"322'3400","created":305,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6851,"log_dups_size":0,"ondisk_log_size":6851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:46.363318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153812992,"num_objects":2347,"num_object_clones":0,"num_object_copies":7041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10251,"num_write_kb":403136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611587+0000","last_change":"2024-09-21T10:22:04.030341+0000","last_active":"2024-09-21T10:22:49.611587+0000","last_peered":"2024-09-21T10:22:49.611587+0000","last_clean":"2024-09-21T10:22:49.611587+0000","last_became_active":"2024-09-21T10:22:04.030000+0000","last_became_peered":"2024-09-21T10:22:04.030000+0000","last_unstale":"2024-09-21T10:22:49.611587+0000","last_undegraded":"2024-09-21T10:22:49.611587+0000","last_fullsized":"2024-09-21T10:22:49.611587+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:10.332770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:10.332770+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1142,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.928149+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:23:06.928149+0000","last_peered":"2024-09-21T10:23:06.928149+0000","last_clean":"2024-09-21T10:23:06.928149+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:23:06.928149+0000","last_undegraded":"2024-09-21T10:23:06.928149+0000","last_fullsized":"2024-09-21T10:23:06.928149+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":301,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.917928+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:23:06.917928+0000","last_peered":"2024-09-21T10:23:06.917928+0000","last_clean":"2024-09-21T10:23:06.917928+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:23:06.917928+0000","last_undegraded":"2024-09-21T10:23:06.917928+0000","last_fullsized":"2024-09-21T10:23:06.917928+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":330,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.918941+0000","last_change":"2024-09-21T10:22:01.317933+0000","last_active":"2024-09-21T10:23:06.918941+0000","last_peered":"2024-09-21T10:23:06.918941+0000","last_clean":"2024-09-21T10:23:06.918941+0000","last_became_active":"2024-09-21T10:22:01.317536+0000","last_became_peered":"2024-09-21T10:22:01.317536+0000","last_unstale":"2024-09-21T10:23:06.918941+0000","last_undegraded":"2024-09-21T10:23:06.918941+0000","last_fullsized":"2024-09-21T10:23:06.918941+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:34.312724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"358'10358","reported_seq":17562,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:13.837188+0000","last_change":"2024-09-21T10:21:34.868559+0000","last_active":"2024-09-21T10:23:13.837188+0000","last_peered":"2024-09-21T10:23:13.837188+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.463996+0000","last_became_peered":"2024-09-21T10:20:14.463996+0000","last_unstale":"2024-09-21T10:23:13.837188+0000","last_undegraded":"2024-09-21T10:20:14.217266+0000","last_fullsized":"2024-09-21T10:20:14.071353+0000","mapping_epoch":325,"log_start":"322'3500","ondisk_log_start":"322'3500","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6858,"log_dups_size":0,"ondisk_log_size":6858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151519232,"num_objects":2312,"num_object_clones":0,"num_object_copies":6936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":98,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10358,"num_write_kb":405440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":232128512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2214}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611530+0000","last_change":"2024-09-21T10:22:04.029892+0000","last_active":"2024-09-21T10:22:49.611530+0000","last_peered":"2024-09-21T10:22:49.611530+0000","last_clean":"2024-09-21T10:22:49.611530+0000","last_became_active":"2024-09-21T10:22:04.029353+0000","last_became_peered":"2024-09-21T10:22:04.029353+0000","last_unstale":"2024-09-21T10:22:49.611530+0000","last_undegraded":"2024-09-21T10:22:49.611530+0000","last_fullsized":"2024-09-21T10:22:49.611530+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:19:13.371825+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1100,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606210+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:22:49.606210+0000","last_peered":"2024-09-21T10:22:49.606210+0000","last_clean":"2024-09-21T10:22:49.606210+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:22:49.606210+0000","last_undegraded":"2024-09-21T10:22:49.606210+0000","last_fullsized":"2024-09-21T10:22:49.606210+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":54,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.607206+0000","last_change":"2024-09-21T10:19:50.581529+0000","last_active":"2024-09-21T10:22:49.607206+0000","last_peered":"2024-09-21T10:22:49.607206+0000","last_clean":"2024-09-21T10:22:49.607206+0000","last_became_active":"2024-09-21T10:19:50.581119+0000","last_became_peered":"2024-09-21T10:19:50.581119+0000","last_unstale":"2024-09-21T10:22:49.607206+0000","last_undegraded":"2024-09-21T10:22:49.607206+0000","last_fullsized":"2024-09-21T10:22:49.607206+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:13.334483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":300,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.918119+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:23:06.918119+0000","last_peered":"2024-09-21T10:23:06.918119+0000","last_clean":"2024-09-21T10:23:06.918119+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:23:06.918119+0000","last_undegraded":"2024-09-21T10:23:06.918119+0000","last_fullsized":"2024-09-21T10:23:06.918119+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"358'10351","reported_seq":14457,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:11.655287+0000","last_change":"2024-09-21T10:20:42.601607+0000","last_active":"2024-09-21T10:23:11.655287+0000","last_peered":"2024-09-21T10:23:11.655287+0000","last_clean":"2024-09-21T10:23:11.655287+0000","last_became_active":"2024-09-21T10:20:42.593406+0000","last_became_peered":"2024-09-21T10:20:42.593406+0000","last_unstale":"2024-09-21T10:23:11.655287+0000","last_undegraded":"2024-09-21T10:23:11.655287+0000","last_fullsized":"2024-09-21T10:23:11.655287+0000","mapping_epoch":330,"log_start":"319'3168","ondisk_log_start":"319'3168","created":305,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7183,"log_dups_size":0,"ondisk_log_size":7183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:09.770489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155385856,"num_objects":2371,"num_object_clones":0,"num_object_copies":7113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10351,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1985,"num_bytes_recovered":130088960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.611872+0000","last_change":"2024-09-21T10:22:04.029371+0000","last_active":"2024-09-21T10:22:49.611872+0000","last_peered":"2024-09-21T10:22:49.611872+0000","last_clean":"2024-09-21T10:22:49.611872+0000","last_became_active":"2024-09-21T10:22:04.029014+0000","last_became_peered":"2024-09-21T10:22:04.029014+0000","last_unstale":"2024-09-21T10:22:49.611872+0000","last_undegraded":"2024-09-21T10:22:49.611872+0000","last_fullsized":"2024-09-21T10:22:49.611872+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":305,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.636199+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:22:49.636199+0000","last_peered":"2024-09-21T10:22:49.636199+0000","last_clean":"2024-09-21T10:22:49.636199+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:22:49.636199+0000","last_undegraded":"2024-09-21T10:22:49.636199+0000","last_fullsized":"2024-09-21T10:22:49.636199+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":909,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.918325+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:23:06.918325+0000","last_peered":"2024-09-21T10:23:06.918325+0000","last_clean":"2024-09-21T10:23:06.918325+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:23:06.918325+0000","last_undegraded":"2024-09-21T10:23:06.918325+0000","last_fullsized":"2024-09-21T10:23:06.918325+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":336,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.919889+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:23:06.919889+0000","last_peered":"2024-09-21T10:23:06.919889+0000","last_clean":"2024-09-21T10:23:06.919889+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:23:06.919889+0000","last_undegraded":"2024-09-21T10:23:06.919889+0000","last_fullsized":"2024-09-21T10:23:06.919889+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"358'10135","reported_seq":10253,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:13.847112+0000","last_change":"2024-09-21T10:19:22.641043+0000","last_active":"2024-09-21T10:23:13.847112+0000","last_peered":"2024-09-21T10:23:13.847112+0000","last_clean":"2024-09-21T10:23:13.847112+0000","last_became_active":"2024-09-21T10:19:22.640742+0000","last_became_peered":"2024-09-21T10:19:22.640742+0000","last_unstale":"2024-09-21T10:23:13.847112+0000","last_undegraded":"2024-09-21T10:23:13.847112+0000","last_fullsized":"2024-09-21T10:23:13.847112+0000","mapping_epoch":305,"log_start":"322'3300","ondisk_log_start":"322'3300","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:36.901208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144900096,"num_objects":2211,"num_object_clones":0,"num_object_copies":6633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10135,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.612542+0000","last_change":"2024-09-21T10:22:04.031872+0000","last_active":"2024-09-21T10:22:49.612542+0000","last_peered":"2024-09-21T10:22:49.612542+0000","last_clean":"2024-09-21T10:22:49.612542+0000","last_became_active":"2024-09-21T10:22:04.031343+0000","last_became_peered":"2024-09-21T10:22:04.031343+0000","last_unstale":"2024-09-21T10:22:49.612542+0000","last_undegraded":"2024-09-21T10:22:49.612542+0000","last_fullsized":"2024-09-21T10:22:49.612542+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:07.332404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:07.332404+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":318,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.918242+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:23:06.918242+0000","last_peered":"2024-09-21T10:23:06.918242+0000","last_clean":"2024-09-21T10:23:06.918242+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:23:06.918242+0000","last_undegraded":"2024-09-21T10:23:06.918242+0000","last_fullsized":"2024-09-21T10:23:06.918242+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":924,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.613225+0000","last_change":"2024-09-21T10:22:01.336961+0000","last_active":"2024-09-21T10:22:49.613225+0000","last_peered":"2024-09-21T10:22:49.613225+0000","last_clean":"2024-09-21T10:22:49.613225+0000","last_became_active":"2024-09-21T10:22:01.320079+0000","last_became_peered":"2024-09-21T10:22:01.320079+0000","last_unstale":"2024-09-21T10:22:49.613225+0000","last_undegraded":"2024-09-21T10:22:49.613225+0000","last_fullsized":"2024-09-21T10:22:49.613225+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:54.037388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":202,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.921311+0000","last_change":"2024-09-21T10:20:13.088557+0000","last_active":"2024-09-21T10:23:06.921311+0000","last_peered":"2024-09-21T10:23:06.921311+0000","last_clean":"2024-09-21T10:23:06.921311+0000","last_became_active":"2024-09-21T10:20:13.088123+0000","last_became_peered":"2024-09-21T10:20:13.088123+0000","last_unstale":"2024-09-21T10:23:06.921311+0000","last_undegraded":"2024-09-21T10:23:06.921311+0000","last_fullsized":"2024-09-21T10:23:06.921311+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:49.475499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"358'10312","reported_seq":15052,"reported_epoch":358,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:23:13.762072+0000","last_change":"2024-09-21T10:22:01.405956+0000","last_active":"2024-09-21T10:23:13.762072+0000","last_peered":"2024-09-21T10:23:13.762072+0000","last_clean":"2024-09-21T10:20:12.067444+0000","last_became_active":"2024-09-21T10:22:01.391808+0000","last_became_peered":"2024-09-21T10:22:01.391808+0000","last_unstale":"2024-09-21T10:23:13.762072+0000","last_undegraded":"2024-09-21T10:22:01.123056+0000","last_fullsized":"2024-09-21T10:23:13.762072+0000","mapping_epoch":346,"log_start":"322'3400","ondisk_log_start":"322'3400","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6912,"log_dups_size":0,"ondisk_log_size":6912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147062784,"num_objects":2244,"num_object_clones":0,"num_object_copies":6732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1413,"num_objects_misplaced":3075,"num_objects_unfound":0,"num_objects_dirty":2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10312,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2300,"num_bytes_recovered":150601728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[3,5,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,5","objects":2244}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.612910+0000","last_change":"2024-09-21T10:22:04.032207+0000","last_active":"2024-09-21T10:22:49.612910+0000","last_peered":"2024-09-21T10:22:49.612910+0000","last_clean":"2024-09-21T10:22:49.612910+0000","last_became_active":"2024-09-21T10:22:04.031881+0000","last_became_peered":"2024-09-21T10:22:04.031881+0000","last_unstale":"2024-09-21T10:22:49.612910+0000","last_undegraded":"2024-09-21T10:22:49.612910+0000","last_fullsized":"2024-09-21T10:22:49.612910+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:08.332410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:08.332410+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":626,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.909263+0000","last_change":"2024-09-21T10:21:34.876196+0000","last_active":"2024-09-21T10:23:06.909263+0000","last_peered":"2024-09-21T10:23:06.909263+0000","last_clean":"2024-09-21T10:23:06.909263+0000","last_became_active":"2024-09-21T10:21:34.868031+0000","last_became_peered":"2024-09-21T10:21:34.868031+0000","last_unstale":"2024-09-21T10:23:06.909263+0000","last_undegraded":"2024-09-21T10:23:06.909263+0000","last_fullsized":"2024-09-21T10:23:06.909263+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:58.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":972,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.918377+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:23:06.918377+0000","last_peered":"2024-09-21T10:23:06.918377+0000","last_clean":"2024-09-21T10:23:06.918377+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:23:06.918377+0000","last_undegraded":"2024-09-21T10:23:06.918377+0000","last_fullsized":"2024-09-21T10:23:06.918377+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":622,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.909789+0000","last_change":"2024-09-21T10:22:01.329400+0000","last_active":"2024-09-21T10:23:06.909789+0000","last_peered":"2024-09-21T10:23:06.909789+0000","last_clean":"2024-09-21T10:23:06.909789+0000","last_became_active":"2024-09-21T10:22:01.328887+0000","last_became_peered":"2024-09-21T10:22:01.328887+0000","last_unstale":"2024-09-21T10:23:06.909789+0000","last_undegraded":"2024-09-21T10:23:06.909789+0000","last_fullsized":"2024-09-21T10:23:06.909789+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:09.626637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"358'10241","reported_seq":15790,"reported_epoch":358,"state":"active+recovering+remapped","last_fresh":"2024-09-21T10:23:14.964449+0000","last_change":"2024-09-21T10:23:07.408964+0000","last_active":"2024-09-21T10:23:14.964449+0000","last_peered":"2024-09-21T10:23:14.964449+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:21:39.424206+0000","last_became_peered":"2024-09-21T10:21:39.424206+0000","last_unstale":"2024-09-21T10:23:14.964449+0000","last_undegraded":"2024-09-21T10:23:14.964449+0000","last_fullsized":"2024-09-21T10:23:14.964449+0000","mapping_epoch":339,"log_start":"313'300","ondisk_log_start":"313'300","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":9941,"log_dups_size":0,"ondisk_log_size":9941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:54.426685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139460632,"num_objects":2129,"num_object_clones":0,"num_object_copies":6387,"num_objects_missing_on_primary":934,"num_objects_missing":934,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2129,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10241,"num_write_kb":395777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2724,"num_bytes_recovered":144637952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,5,6","objects":2129}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":587,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:22:49.612075+0000","last_change":"2024-09-21T10:22:02.009815+0000","last_active":"2024-09-21T10:22:49.612075+0000","last_peered":"2024-09-21T10:22:49.612075+0000","last_clean":"2024-09-21T10:12:59.519603+0000","last_became_active":"2024-09-21T10:22:02.009463+0000","last_became_peered":"2024-09-21T10:22:02.009463+0000","last_unstale":"2024-09-21T10:22:49.612075+0000","last_undegraded":"2024-09-21T10:22:02.002510+0000","last_fullsized":"2024-09-21T10:22:02.001835+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:02.435487+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:02.435487+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1},{"shards":"6,7","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":31,"reported_epoch":358,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:23:06.921604+0000","last_change":"2024-09-21T10:22:01.318966+0000","last_active":"2024-09-21T10:23:06.921604+0000","last_peered":"2024-09-21T10:23:06.921604+0000","last_clean":"2024-09-21T10:21:59.002104+0000","last_became_active":"2024-09-21T10:22:01.317312+0000","last_became_peered":"2024-09-21T10:22:01.317312+0000","last_unstale":"2024-09-21T10:23:06.921604+0000","last_undegraded":"2024-09-21T10:22:00.997012+0000","last_fullsized":"2024-09-21T10:23:06.921604+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":643,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:22:49.611127+0000","last_change":"2024-09-21T10:21:34.866048+0000","last_active":"2024-09-21T10:22:49.611127+0000","last_peered":"2024-09-21T10:22:49.611127+0000","last_clean":"2024-09-21T10:21:17.726080+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:22:49.611127+0000","last_undegraded":"2024-09-21T10:21:34.856315+0000","last_fullsized":"2024-09-21T10:22:49.611127+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":330,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606433+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:22:49.606433+0000","last_peered":"2024-09-21T10:22:49.606433+0000","last_clean":"2024-09-21T10:22:49.606433+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:22:49.606433+0000","last_undegraded":"2024-09-21T10:22:49.606433+0000","last_fullsized":"2024-09-21T10:22:49.606433+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":355,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:23:05.009139+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:23:05.009139+0000","last_peered":"2024-09-21T10:23:05.009139+0000","last_clean":"2024-09-21T10:23:05.009139+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:23:05.009139+0000","last_undegraded":"2024-09-21T10:23:05.009139+0000","last_fullsized":"2024-09-21T10:23:05.009139+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":64,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606913+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:22:49.606913+0000","last_peered":"2024-09-21T10:22:49.606913+0000","last_clean":"2024-09-21T10:22:49.606913+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:22:49.606913+0000","last_undegraded":"2024-09-21T10:22:49.606913+0000","last_fullsized":"2024-09-21T10:22:49.606913+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606197+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:22:49.606197+0000","last_peered":"2024-09-21T10:22:49.606197+0000","last_clean":"2024-09-21T10:22:49.606197+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:22:49.606197+0000","last_undegraded":"2024-09-21T10:22:49.606197+0000","last_fullsized":"2024-09-21T10:22:49.606197+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":36,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.612702+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:22:49.612702+0000","last_peered":"2024-09-21T10:22:49.612702+0000","last_clean":"2024-09-21T10:22:49.612702+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:22:49.612702+0000","last_undegraded":"2024-09-21T10:22:49.612702+0000","last_fullsized":"2024-09-21T10:22:49.612702+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":65,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606108+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:22:49.606108+0000","last_peered":"2024-09-21T10:22:49.606108+0000","last_clean":"2024-09-21T10:22:49.606108+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:22:49.606108+0000","last_undegraded":"2024-09-21T10:22:49.606108+0000","last_fullsized":"2024-09-21T10:22:49.606108+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":546,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.908507+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:23:06.908507+0000","last_peered":"2024-09-21T10:23:06.908507+0000","last_clean":"2024-09-21T10:23:06.908507+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:23:06.908507+0000","last_undegraded":"2024-09-21T10:23:06.908507+0000","last_fullsized":"2024-09-21T10:23:06.908507+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":201,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.921859+0000","last_change":"2024-09-21T10:20:13.089473+0000","last_active":"2024-09-21T10:23:06.921859+0000","last_peered":"2024-09-21T10:23:06.921859+0000","last_clean":"2024-09-21T10:23:06.921859+0000","last_became_active":"2024-09-21T10:20:13.089111+0000","last_became_peered":"2024-09-21T10:20:13.089111+0000","last_unstale":"2024-09-21T10:23:06.921859+0000","last_undegraded":"2024-09-21T10:23:06.921859+0000","last_fullsized":"2024-09-21T10:23:06.921859+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:06.603284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":606,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.909702+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:23:06.909702+0000","last_peered":"2024-09-21T10:23:06.909702+0000","last_clean":"2024-09-21T10:23:06.909702+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:23:06.909702+0000","last_undegraded":"2024-09-21T10:23:06.909702+0000","last_fullsized":"2024-09-21T10:23:06.909702+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":371,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.614141+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:22:49.614141+0000","last_peered":"2024-09-21T10:22:49.614141+0000","last_clean":"2024-09-21T10:22:49.614141+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:22:49.614141+0000","last_undegraded":"2024-09-21T10:22:49.614141+0000","last_fullsized":"2024-09-21T10:22:49.614141+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":623,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.909913+0000","last_change":"2024-09-21T10:22:01.329690+0000","last_active":"2024-09-21T10:23:06.909913+0000","last_peered":"2024-09-21T10:23:06.909913+0000","last_clean":"2024-09-21T10:23:06.909913+0000","last_became_active":"2024-09-21T10:22:01.329033+0000","last_became_peered":"2024-09-21T10:22:01.329033+0000","last_unstale":"2024-09-21T10:23:06.909913+0000","last_undegraded":"2024-09-21T10:23:06.909913+0000","last_fullsized":"2024-09-21T10:23:06.909913+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:24.559524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":205,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.921766+0000","last_change":"2024-09-21T10:21:34.870704+0000","last_active":"2024-09-21T10:23:06.921766+0000","last_peered":"2024-09-21T10:23:06.921766+0000","last_clean":"2024-09-21T10:23:06.921766+0000","last_became_active":"2024-09-21T10:21:34.869835+0000","last_became_peered":"2024-09-21T10:21:34.869835+0000","last_unstale":"2024-09-21T10:23:06.921766+0000","last_undegraded":"2024-09-21T10:23:06.921766+0000","last_fullsized":"2024-09-21T10:23:06.921766+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:34.959915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":321,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.917866+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:23:06.917866+0000","last_peered":"2024-09-21T10:23:06.917866+0000","last_clean":"2024-09-21T10:23:06.917866+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:23:06.917866+0000","last_undegraded":"2024-09-21T10:23:06.917866+0000","last_fullsized":"2024-09-21T10:23:06.917866+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":612,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.908756+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:23:06.908756+0000","last_peered":"2024-09-21T10:23:06.908756+0000","last_clean":"2024-09-21T10:23:06.908756+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:23:06.908756+0000","last_undegraded":"2024-09-21T10:23:06.908756+0000","last_fullsized":"2024-09-21T10:23:06.908756+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":362,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.998861+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:22:49.998861+0000","last_peered":"2024-09-21T10:22:49.998861+0000","last_clean":"2024-09-21T10:22:49.998861+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:22:49.998861+0000","last_undegraded":"2024-09-21T10:22:49.998861+0000","last_fullsized":"2024-09-21T10:22:49.998861+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":611,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.928894+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:23:06.928894+0000","last_peered":"2024-09-21T10:23:06.928894+0000","last_clean":"2024-09-21T10:23:06.928894+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:23:06.928894+0000","last_undegraded":"2024-09-21T10:23:06.928894+0000","last_fullsized":"2024-09-21T10:23:06.928894+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":34,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.921249+0000","last_change":"2024-09-21T10:22:01.318549+0000","last_active":"2024-09-21T10:23:06.921249+0000","last_peered":"2024-09-21T10:23:06.921249+0000","last_clean":"2024-09-21T10:23:06.921249+0000","last_became_active":"2024-09-21T10:22:01.318071+0000","last_became_peered":"2024-09-21T10:22:01.318071+0000","last_unstale":"2024-09-21T10:23:06.921249+0000","last_undegraded":"2024-09-21T10:23:06.921249+0000","last_fullsized":"2024-09-21T10:23:06.921249+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:49.061492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":54,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.607088+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:22:49.607088+0000","last_peered":"2024-09-21T10:22:49.607088+0000","last_clean":"2024-09-21T10:22:49.607088+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:22:49.607088+0000","last_undegraded":"2024-09-21T10:22:49.607088+0000","last_fullsized":"2024-09-21T10:22:49.607088+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":297,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606503+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:22:49.606503+0000","last_peered":"2024-09-21T10:22:49.606503+0000","last_clean":"2024-09-21T10:22:49.606503+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:22:49.606503+0000","last_undegraded":"2024-09-21T10:22:49.606503+0000","last_fullsized":"2024-09-21T10:22:49.606503+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":54,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.606552+0000","last_change":"2024-09-21T10:19:50.588363+0000","last_active":"2024-09-21T10:22:49.606552+0000","last_peered":"2024-09-21T10:22:49.606552+0000","last_clean":"2024-09-21T10:22:49.606552+0000","last_became_active":"2024-09-21T10:19:50.588107+0000","last_became_peered":"2024-09-21T10:19:50.588107+0000","last_unstale":"2024-09-21T10:22:49.606552+0000","last_undegraded":"2024-09-21T10:22:49.606552+0000","last_fullsized":"2024-09-21T10:22:49.606552+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:05.698018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":275,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.636256+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:22:49.636256+0000","last_peered":"2024-09-21T10:22:49.636256+0000","last_clean":"2024-09-21T10:22:49.636256+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:22:49.636256+0000","last_undegraded":"2024-09-21T10:22:49.636256+0000","last_fullsized":"2024-09-21T10:22:49.636256+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":626,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.908888+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:23:06.908888+0000","last_peered":"2024-09-21T10:23:06.908888+0000","last_clean":"2024-09-21T10:23:06.908888+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:23:06.908888+0000","last_undegraded":"2024-09-21T10:23:06.908888+0000","last_fullsized":"2024-09-21T10:23:06.908888+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":587,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-21T10:23:06.909156+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:23:06.909156+0000","last_peered":"2024-09-21T10:23:06.909156+0000","last_clean":"2024-09-21T10:23:06.909156+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:23:06.909156+0000","last_undegraded":"2024-09-21T10:23:06.909156+0000","last_fullsized":"2024-09-21T10:23:06.909156+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":54,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-21T10:22:49.605780+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:22:49.605780+0000","last_peered":"2024-09-21T10:22:49.605780+0000","last_clean":"2024-09-21T10:22:49.605780+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:22:49.605780+0000","last_undegraded":"2024-09-21T10:22:49.605780+0000","last_fullsized":"2024-09-21T10:22:49.605780+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":31,"reported_epoch":358,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:23:06.921744+0000","last_change":"2024-09-21T10:22:01.317167+0000","last_active":"2024-09-21T10:23:06.921744+0000","last_peered":"2024-09-21T10:23:06.921744+0000","last_clean":"2024-09-21T10:14:43.015500+0000","last_became_active":"2024-09-21T10:22:01.316569+0000","last_became_peered":"2024-09-21T10:22:01.316569+0000","last_unstale":"2024-09-21T10:23:06.921744+0000","last_undegraded":"2024-09-21T10:22:00.997069+0000","last_fullsized":"2024-09-21T10:23:06.921744+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2342518808,"num_objects":35745,"num_object_clones":0,"num_object_copies":107235,"num_objects_missing_on_primary":934,"num_objects_missing":934,"num_objects_degraded":14796,"num_objects_misplaced":5223,"num_objects_unfound":0,"num_objects_dirty":35745,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163941,"num_write_kb":6389889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22832,"num_bytes_recovered":1332543488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7707897856,"data_stored":7707885640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107912,"ondisk_log_size":107912,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":3,"num_objects_missing":3,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24225,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":391,"num_read_kb":833,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":3107188,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1640,"ondisk_log_size":1640,"up":33,"acting":31,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234403,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1029448,"kb_used_data":438848,"kb_used_omap":1,"kb_used_meta":590590,"kb_avail":92715704,"statfs":{"total":95995035648,"available":94940880896,"internally_reserved":0,"allocated":449380352,"data_stored":447539120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":604764619},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496041,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1942544,"kb_used_data":1325704,"kb_used_omap":1,"kb_used_meta":616830,"kb_avail":91802608,"statfs":{"total":95995035648,"available":94005870592,"internally_reserved":0,"allocated":1357520896,"data_stored":1355672826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":631634375},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403941,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1557376,"kb_used_data":1303032,"kb_used_omap":7,"kb_used_meta":254264,"kb_avail":92187776,"statfs":{"total":95995035648,"available":94400282624,"internally_reserved":0,"allocated":1334304768,"data_stored":1332467782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":260367256},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019314,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1456968,"kb_used_data":866240,"kb_used_omap":1,"kb_used_meta":590590,"kb_avail":92288184,"statfs":{"total":95995035648,"available":94503100416,"internally_reserved":0,"allocated":887029760,"data_stored":885188789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":604764614},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435498,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2022644,"kb_used_data":1255148,"kb_used_omap":5,"kb_used_meta":767482,"kb_avail":91722508,"statfs":{"total":95995035648,"available":93923848192,"internally_reserved":0,"allocated":1285271552,"data_stored":1283446261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":785901608},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599025,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1535868,"kb_used_data":845940,"kb_used_omap":7,"kb_used_meta":689720,"kb_avail":92209284,"statfs":{"total":95995035648,"available":94422306816,"internally_reserved":0,"allocated":866242560,"data_stored":864409863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":706273741},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762552,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1627384,"kb_used_data":1005680,"kb_used_omap":7,"kb_used_meta":621688,"kb_avail":92117768,"statfs":{"total":95995035648,"available":94328594432,"internally_reserved":0,"allocated":1029816320,"data_stored":1027987155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":636608765},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926078,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1088500,"kb_used_data":627820,"kb_used_omap":7,"kb_used_meta":460664,"kb_avail":92656652,"statfs":{"total":95995035648,"available":94880411648,"internally_reserved":0,"allocated":642887680,"data_stored":641062321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":471720126},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":621805568,"data_stored":621805568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":998965248,"data_stored":998965248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":860950528,"data_stored":860946456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1277820928,"data_stored":1277820928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":868614144,"data_stored":868614144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1299058688,"data_stored":1299054616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1344802816,"data_stored":1344798744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":435879936,"data_stored":435879936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:23:18.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:23:18.605 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:18.655 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:18.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:18.757 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:18.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.805 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:18.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:18.906 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:18.939 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:18.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:19.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:19.053 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:23:19.076 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:19.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:19.219 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:19.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:19.321 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:19.360 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:19.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:19.462 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:19.505 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:19.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:19.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:19.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:19.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:19.749 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:19.786 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:19 smithi045 ceph-mon[95976]: pgmap v1081: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 14821/105633 objects degraded (14.031%); 5223/105633 objects misplaced (4.944%); 2.4 MiB/s, 82 objects/s recovering 2024-09-21T10:23:19.786 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:19.832 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:19.832 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:19.833 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:19 smithi032 ceph-mon[105748]: pgmap v1081: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 14821/105633 objects degraded (14.031%); 5223/105633 objects misplaced (4.944%); 2.4 MiB/s, 82 objects/s recovering 2024-09-21T10:23:19.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:19.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:19.833 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:19 smithi032 ceph-mon[101299]: pgmap v1081: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 14821/105633 objects degraded (14.031%); 5223/105633 objects misplaced (4.944%); 2.4 MiB/s, 82 objects/s recovering 2024-09-21T10:23:19.833 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:23:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:23:19] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-09-21T10:23:19.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:19.929 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:20.031 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:20.073 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:20.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:20.174 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:20.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:20.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:20.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:20.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:20.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:20.449 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:20.485 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:20.587 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:20.620 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:20.722 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:20.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:20.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:20.897 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:20.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:20.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:21.042 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:21.144 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:21.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:21.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:21.288 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:21.290 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:23:21.328 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:21.430 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:21.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:21.520 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:21.541 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:21.570 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:21.596 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:21 smithi032 ceph-mon[101299]: pgmap v1082: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 14821/94473 objects degraded (15.688%); 4351/94473 objects misplaced (4.606%); 2.3 MiB/s, 108 objects/s recovering 2024-09-21T10:23:21.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.629 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:21.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:21.730 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:21.775 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:21.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:21.854 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:21 smithi032 ceph-mon[105748]: pgmap v1082: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 14821/94473 objects degraded (15.688%); 4351/94473 objects misplaced (4.606%); 2.3 MiB/s, 108 objects/s recovering 2024-09-21T10:23:21.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:21.911 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:22.014 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:21 smithi045 ceph-mon[95976]: pgmap v1082: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 14821/94473 objects degraded (15.688%); 4351/94473 objects misplaced (4.606%); 2.3 MiB/s, 108 objects/s recovering 2024-09-21T10:23:22.066 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:22.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:22.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:22.213 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:22.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:22.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:22.357 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:22.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:22.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:22.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:22.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:22.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.652 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:22.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:22.753 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:22.787 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:22.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:22.889 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:22.903 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14821/94473 objects degraded (15.688%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:22.903 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14821/94473 objects degraded (15.688%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:22.908 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:22.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.928 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:22.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:23.028 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14821/94473 objects degraded (15.688%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:23.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.067 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:23.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:23.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:23.203 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:23.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:23.305 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:23.344 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:23.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:23.445 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:23.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:23.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:23.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:23.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:23.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:23.620 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:23:23.704 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:23.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:23.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:23.841 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:23.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:23.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:23.983 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:23 smithi032 ceph-mon[105748]: pgmap v1083: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 14821/93033 objects degraded (15.931%); 4351/93033 objects misplaced (4.677%); 0 B/s, 59 objects/s recovering 2024-09-21T10:23:23.983 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2349043327' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:23:23.983 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:23 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:23:23.984 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:23 smithi032 ceph-mon[101299]: pgmap v1083: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 14821/93033 objects degraded (15.931%); 4351/93033 objects misplaced (4.677%); 0 B/s, 59 objects/s recovering 2024-09-21T10:23:23.984 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2349043327' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:23:23.984 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:23 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:23:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:23 smithi045 ceph-mon[95976]: pgmap v1083: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 14821/93033 objects degraded (15.931%); 4351/93033 objects misplaced (4.677%); 0 B/s, 59 objects/s recovering 2024-09-21T10:23:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2349043327' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:23:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:23 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:23:24.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:24.115 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:24.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:24.217 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:24.246 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:24.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:24.348 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:24.379 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:24.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:24.481 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:24.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:24.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:24.619 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:24.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.663 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:24.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:24.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:24.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:24.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:24.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:24.936 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:24.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:24.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:24 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:24.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:24 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:23:24.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:24 smithi032 ceph-mon[105748]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T10:23:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:24 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:24 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:23:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:24 smithi032 ceph-mon[101299]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T10:23:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:24 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:24 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:23:25.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:24 smithi045 ceph-mon[95976]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T10:23:25.039 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:25.075 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:25.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:25.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:25.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:25.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:25.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:25.447 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:25.490 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:25.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:25.592 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:25.627 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:25.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:25.767 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:25.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:25.869 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:25.903 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:25.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:25.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:25 smithi032 ceph-mon[101299]: pgmap v1085: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 14821/89175 objects degraded (16.620%); 3851/89175 objects misplaced (4.318%); 0 B/s, 71 objects/s recovering 2024-09-21T10:23:25.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:25 smithi032 ceph-mon[105748]: pgmap v1085: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 14821/89175 objects degraded (16.620%); 3851/89175 objects misplaced (4.318%); 0 B/s, 71 objects/s recovering 2024-09-21T10:23:26.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.034 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:26.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:26.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:25 smithi045 ceph-mon[95976]: pgmap v1085: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 3 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 14821/89175 objects degraded (16.620%); 3851/89175 objects misplaced (4.318%); 0 B/s, 71 objects/s recovering 2024-09-21T10:23:26.136 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:26.167 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:26.269 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:26.304 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:26.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:26.405 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:26.449 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:26.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:26.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:26.581 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:26.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:26.683 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:26.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:26.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:26.818 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:26.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.855 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:26.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:26.956 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:26.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:26 smithi032 ceph-mon[101299]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T10:23:26.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:26 smithi032 ceph-mon[105748]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T10:23:26.983 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:26.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:27.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:26 smithi045 ceph-mon[95976]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T10:23:27.085 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:27.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:27.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:27.209 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:27.237 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:27.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:27.339 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:27.365 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:27.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:27.467 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.493 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:27.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:27.594 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:27.618 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:27.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:27.721 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:27.751 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:27 smithi032 ceph-mon[105748]: pgmap v1087: 99 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 14626/82122 objects degraded (17.810%); 5402/82122 objects misplaced (6.578%); 72 KiB/s, 117 objects/s recovering 2024-09-21T10:23:27.752 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:27 smithi032 ceph-mon[105748]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T10:23:27.752 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:27 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14626/82122 objects degraded (17.810%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:27.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:27 smithi032 ceph-mon[101299]: pgmap v1087: 99 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 14626/82122 objects degraded (17.810%); 5402/82122 objects misplaced (6.578%); 72 KiB/s, 117 objects/s recovering 2024-09-21T10:23:27.753 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:27 smithi032 ceph-mon[101299]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T10:23:27.753 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:27 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14626/82122 objects degraded (17.810%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:27.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:27.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:27.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:27.891 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:27.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:28.027 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:27 smithi045 ceph-mon[95976]: pgmap v1087: 99 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 14626/82122 objects degraded (17.810%); 5402/82122 objects misplaced (6.578%); 72 KiB/s, 117 objects/s recovering 2024-09-21T10:23:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:27 smithi045 ceph-mon[95976]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T10:23:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:27 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14626/82122 objects degraded (17.810%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:28.128 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:28.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:28.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:28.319 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:28.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:28.421 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:28.457 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:28.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:28.558 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:28.593 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:28.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:28.694 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:28.735 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:28.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:28.837 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:28.875 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:28.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:28.909 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:28 smithi032 ceph-mon[101299]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T10:23:28.909 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:28 smithi032 ceph-mon[105748]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T10:23:28.977 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:29.008 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:28 smithi045 ceph-mon[95976]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T10:23:29.110 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:29.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.145 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:29.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:29.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:29.292 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:29.393 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:29.426 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:29.527 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:29.563 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:29.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:29.664 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:29.713 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:29.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:29.804 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:29 smithi032 ceph-mon[101299]: pgmap v1090: 99 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 14623/80691 objects degraded (18.122%); 5402/80691 objects misplaced (6.695%); 117 KiB/s, 121 objects/s recovering 2024-09-21T10:23:29.805 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:29 smithi032 ceph-mon[105748]: pgmap v1090: 99 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 14623/80691 objects degraded (18.122%); 5402/80691 objects misplaced (6.695%); 117 KiB/s, 121 objects/s recovering 2024-09-21T10:23:29.805 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:23:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:23:29] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-09-21T10:23:29.815 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:29.852 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:29.954 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:29.987 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:29.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:29 smithi045 ceph-mon[95976]: pgmap v1090: 99 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 14623/80691 objects degraded (18.122%); 5402/80691 objects misplaced (6.695%); 117 KiB/s, 121 objects/s recovering 2024-09-21T10:23:30.089 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:30.117 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:30.219 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:30.257 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:30.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:30.358 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:30.375 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:23:30.375 INFO:teuthology.orchestra.run.smithi032.stdout:{"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 14623/80691 objects degraded (18.122%), 10 pgs degraded, 8 pgs undersized","count":18},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1500,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":362,"num_osds":8,"num_up_osds":8,"osd_up_since":1726914012,"num_in_osds":8,"osd_in_since":1726913989,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+undersized+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":26897,"data_bytes":1765611517,"bytes_used":10900037632,"bytes_avail":757060247552,"bytes_total":767960285184,"degraded_objects":14623,"degraded_total":80691,"degraded_ratio":0.18122219330532524,"misplaced_objects":5402,"misplaced_total":80691,"misplaced_ratio":0.066946747468738771,"recovering_objects_per_sec":121,"recovering_bytes_per_sec":119721,"recovering_keys_per_sec":0,"num_objects_recovered":601,"num_bytes_recovered":590406,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":834},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi032.fdolcm","status":"up:active","gid":14655}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.32:8443/","prometheus":"http://172.21.15.32:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-21T10:21:12.493782+0000","services":{}},"progress_events":{}} 2024-09-21T10:23:30.397 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:30.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:30.499 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:30.538 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:30.640 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:30.682 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:30.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:30.784 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:30.833 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:30.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:30.935 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:30.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:30 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1010578609' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:23:30.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:30 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1010578609' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:23:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.972 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:30.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:31.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:30 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1010578609' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:23:31.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:31.106 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:31.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:31.208 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:31.244 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:31.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:31.345 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:31.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.390 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:31.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:31.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:23:31.399 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:23:31.428 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:23:31.428 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:23:31.461 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:23:31.461 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:23:31.486 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:23:31.486 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:23:31.492 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:31.531 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:31.532 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:23:31.533 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:23:31.578 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:23:31.578 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:23:31.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:23:31.615 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:23:31.633 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:31.648 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:23:31.648 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:23:31.679 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:31.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:31.781 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:31.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.810 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:31.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:31.869 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:31 smithi032 ceph-mon[105748]: pgmap v1091: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 12924/71067 objects degraded (18.186%); 2242/71067 objects misplaced (3.155%); 8.2 MiB/s, 308 objects/s recovering 2024-09-21T10:23:31.869 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:31 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4265301839' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:31.869 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:31 smithi032 ceph-mon[101299]: pgmap v1091: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 12924/71067 objects degraded (18.186%); 2242/71067 objects misplaced (3.155%); 8.2 MiB/s, 308 objects/s recovering 2024-09-21T10:23:31.869 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:31 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4265301839' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:31.911 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:31.946 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:32.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:31 smithi045 ceph-mon[95976]: pgmap v1091: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 12924/71067 objects degraded (18.186%); 2242/71067 objects misplaced (3.155%); 8.2 MiB/s, 308 objects/s recovering 2024-09-21T10:23:32.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:31 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4265301839' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:32.048 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:32.083 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:32.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:32.226 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:32.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:32.328 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:32.360 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:32.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:32.461 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:32.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:32.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:32.605 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:32.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.650 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:32.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:32.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:32.792 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:32.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:32.853 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:32 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12924/71067 objects degraded (18.186%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:32.853 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:32 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12924/71067 objects degraded (18.186%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:32.894 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:32.930 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:32.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:33.032 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:33.038 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:32 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12924/71067 objects degraded (18.186%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:33.073 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:33.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:33.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:33.220 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:33.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:33.242 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:23:33.242 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:23:33.321 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:33.359 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:33.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:33.461 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:33.496 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:33.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:33.598 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:33.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.634 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:33.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:33.735 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:33.764 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:33.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:33.807 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:33 smithi032 ceph-mon[101299]: pgmap v1092: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 12924/69696 objects degraded (18.543%); 2242/69696 objects misplaced (3.217%); 7.1 MiB/s, 181 objects/s recovering 2024-09-21T10:23:33.807 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:33.808 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:33.808 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:33 smithi032 ceph-mon[105748]: pgmap v1092: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 12924/69696 objects degraded (18.543%); 2242/69696 objects misplaced (3.217%); 7.1 MiB/s, 181 objects/s recovering 2024-09-21T10:23:33.808 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:33.808 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:33.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:33.900 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:33.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:34.002 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:33 smithi045 ceph-mon[95976]: pgmap v1092: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 12924/69696 objects degraded (18.543%); 2242/69696 objects misplaced (3.217%); 7.1 MiB/s, 181 objects/s recovering 2024-09-21T10:23:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:23:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:34.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:34.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:34.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:34.194 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:34.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:34.296 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:34.347 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:34.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:34.436 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:23:34.436 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:23:34.448 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:34.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:34.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:34.602 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.645 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:34.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:34.746 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:34.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:34.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:34.778 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:23:34.879 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:34.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:34.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:35.024 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:35.060 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:35.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:35.162 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:35.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.196 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:35.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:35.296 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:35.298 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:35.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:23:35.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:35.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:35.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:35.470 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:35.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:35.572 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:35.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:35.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:35.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:35.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:35.843 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:23:35.851 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:35.944 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:35.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:35 smithi032 ceph-mon[101299]: pgmap v1093: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 12741/66552 objects degraded (19.144%); 1919/66552 objects misplaced (2.883%); 6.1 MiB/s, 156 objects/s recovering 2024-09-21T10:23:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:35 smithi032 ceph-mon[105748]: pgmap v1093: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 12741/66552 objects degraded (19.144%); 1919/66552 objects misplaced (2.883%); 6.1 MiB/s, 156 objects/s recovering 2024-09-21T10:23:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:35 smithi045 ceph-mon[95976]: pgmap v1093: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 12741/66552 objects degraded (19.144%); 1919/66552 objects misplaced (2.883%); 6.1 MiB/s, 156 objects/s recovering 2024-09-21T10:23:36.046 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:36.073 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:36.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:36.211 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:36.313 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:36.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:36.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:36.448 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:36.485 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:36.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:36.575 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:36.587 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:36.654 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:36.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:36.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:36.905 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:36.950 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:36.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:37.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:37.081 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:37.183 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:37.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:37.327 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:37.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:37.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:37.462 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:37.506 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:37.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:37.608 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:37.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.659 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:37.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:37.760 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:37.778 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:37 smithi032 ceph-mon[101299]: pgmap v1094: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 12741/58527 objects degraded (21.769%); 1792/58527 objects misplaced (3.062%); 4.9 MiB/s, 163 objects/s recovering 2024-09-21T10:23:37.778 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:37 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12741/58527 objects degraded (21.769%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:37.778 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:37 smithi032 ceph-mon[105748]: pgmap v1094: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 12741/58527 objects degraded (21.769%); 1792/58527 objects misplaced (3.062%); 4.9 MiB/s, 163 objects/s recovering 2024-09-21T10:23:37.778 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:37 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12741/58527 objects degraded (21.769%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:37.794 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:37.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:37.895 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:37.927 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:37.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:38.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:38.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:37 smithi045 ceph-mon[95976]: pgmap v1094: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 12741/58527 objects degraded (21.769%); 1792/58527 objects misplaced (3.062%); 4.9 MiB/s, 163 objects/s recovering 2024-09-21T10:23:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:37 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12741/58527 objects degraded (21.769%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.062 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:38.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:38.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:38.198 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:38.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:38.334 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:38.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:38.435 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:38.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:38.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:38.607 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:38.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:38.722 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:38.740 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:38.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:38.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:38.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.893 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:38.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:38.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:39.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:39.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:39.115 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:23:39.117 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:23:39.143 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:39.181 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:39.282 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:39.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.339 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:39.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:39.440 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:39.475 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:39.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:39.577 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:39.612 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:39.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:39.680 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:23:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:23:39] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-09-21T10:23:39.714 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:39.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:39.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:39.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:39.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:39.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:39 smithi032 ceph-mon[105748]: pgmap v1095: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 12741/57006 objects degraded (22.350%); 1792/57006 objects misplaced (3.144%); 4.5 MiB/s, 148 objects/s recovering 2024-09-21T10:23:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:39 smithi032 ceph-mon[105748]: from='client.35838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:23:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:39 smithi032 ceph-mon[101299]: pgmap v1095: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 12741/57006 objects degraded (22.350%); 1792/57006 objects misplaced (3.144%); 4.5 MiB/s, 148 objects/s recovering 2024-09-21T10:23:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:39 smithi032 ceph-mon[101299]: from='client.35838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:23:39.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:40.015 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:40.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:39 smithi045 ceph-mon[95976]: pgmap v1095: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 12741/57006 objects degraded (22.350%); 1792/57006 objects misplaced (3.144%); 4.5 MiB/s, 148 objects/s recovering 2024-09-21T10:23:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:39 smithi045 ceph-mon[95976]: from='client.35838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:23:40.117 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:40.159 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:40.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:40.194 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1095,"stamp":"2024-09-21T10:23:38.544088+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1248204797,"num_objects":19002,"num_object_clones":0,"num_object_copies":57006,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":12741,"num_objects_misplaced":1792,"num_objects_unfound":0,"num_objects_dirty":19002,"num_whiteouts":0,"num_read":6389,"num_read_kb":13844,"num_write":181835,"num_write_kb":6404930,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25670,"num_bytes_recovered":1404207679,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112156,"ondisk_log_size":112156,"up":297,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9155328,"kb_used_data":4510784,"kb_used_omap":40,"kb_used_meta":4644375,"kb_avail":740805888,"statfs":{"total":767960285184,"available":758585229312,"internally_reserved":0,"allocated":4619042816,"data_stored":4604110515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41824,"internal_metadata":4755840160},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-517406720,"num_objects":-7895,"num_object_clones":0,"num_object_copies":-23685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1882,"num_objects_misplaced":-3610,"num_objects_unfound":0,"num_objects_dirty":-7895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7895,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1618,"num_bytes_recovered":51183616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.888538"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":601,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.657368+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:23:27.657368+0000","last_peered":"2024-09-21T10:23:27.657368+0000","last_clean":"2024-09-21T10:23:27.657368+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:23:27.657368+0000","last_undegraded":"2024-09-21T10:23:27.657368+0000","last_fullsized":"2024-09-21T10:23:27.657368+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":365,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.662050+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:23:26.662050+0000","last_peered":"2024-09-21T10:23:26.662050+0000","last_clean":"2024-09-21T10:23:26.662050+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:23:26.662050+0000","last_undegraded":"2024-09-21T10:23:26.662050+0000","last_fullsized":"2024-09-21T10:23:26.662050+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.665626+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:23:26.665626+0000","last_peered":"2024-09-21T10:23:26.665626+0000","last_clean":"2024-09-21T10:23:26.665626+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:23:26.665626+0000","last_undegraded":"2024-09-21T10:23:26.665626+0000","last_fullsized":"2024-09-21T10:23:26.665626+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":42,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712237+0000","last_change":"2024-09-21T10:22:01.318638+0000","last_active":"2024-09-21T10:23:27.712237+0000","last_peered":"2024-09-21T10:23:27.712237+0000","last_clean":"2024-09-21T10:23:27.712237+0000","last_became_active":"2024-09-21T10:22:01.317087+0000","last_became_peered":"2024-09-21T10:22:01.317087+0000","last_unstale":"2024-09-21T10:23:27.712237+0000","last_undegraded":"2024-09-21T10:23:27.712237+0000","last_fullsized":"2024-09-21T10:23:27.712237+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:25.334726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":43,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712517+0000","last_change":"2024-09-21T10:22:01.318978+0000","last_active":"2024-09-21T10:23:27.712517+0000","last_peered":"2024-09-21T10:23:27.712517+0000","last_clean":"2024-09-21T10:23:27.712517+0000","last_became_active":"2024-09-21T10:22:01.318255+0000","last_became_peered":"2024-09-21T10:22:01.318255+0000","last_unstale":"2024-09-21T10:23:27.712517+0000","last_undegraded":"2024-09-21T10:23:27.712517+0000","last_fullsized":"2024-09-21T10:23:27.712517+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:55.084088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":336,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.661429+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:23:26.661429+0000","last_peered":"2024-09-21T10:23:26.661429+0000","last_clean":"2024-09-21T10:23:26.661429+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:23:26.661429+0000","last_undegraded":"2024-09-21T10:23:26.661429+0000","last_fullsized":"2024-09-21T10:23:26.661429+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":622,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.658239+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:23:27.658239+0000","last_peered":"2024-09-21T10:23:27.658239+0000","last_clean":"2024-09-21T10:23:27.658239+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:23:27.658239+0000","last_undegraded":"2024-09-21T10:23:27.658239+0000","last_fullsized":"2024-09-21T10:23:27.658239+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":417,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.664633+0000","last_change":"2024-09-21T10:22:01.331612+0000","last_active":"2024-09-21T10:23:26.664633+0000","last_peered":"2024-09-21T10:23:26.664633+0000","last_clean":"2024-09-21T10:23:26.664633+0000","last_became_active":"2024-09-21T10:22:01.325282+0000","last_became_peered":"2024-09-21T10:22:01.325282+0000","last_unstale":"2024-09-21T10:23:26.664633+0000","last_undegraded":"2024-09-21T10:23:26.664633+0000","last_fullsized":"2024-09-21T10:23:26.664633+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:23.383106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"362'11290","reported_seq":12358,"reported_epoch":362,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:35.078038+0000","last_change":"2024-09-21T10:21:39.510930+0000","last_active":"2024-09-21T10:23:35.078038+0000","last_peered":"2024-09-21T10:23:35.078038+0000","last_clean":"2024-09-21T10:20:12.056605+0000","last_became_active":"2024-09-21T10:20:14.466147+0000","last_became_peered":"2024-09-21T10:20:14.466147+0000","last_unstale":"2024-09-21T10:23:35.078038+0000","last_undegraded":"2024-09-21T10:20:14.094800+0000","last_fullsized":"2024-09-21T10:20:14.071992+0000","mapping_epoch":325,"log_start":"326'3900","ondisk_log_start":"326'3900","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7390,"log_dups_size":0,"ondisk_log_size":7390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:34.237367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80609280,"num_objects":1230,"num_object_clones":0,"num_object_copies":3690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3167,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11290,"num_write_kb":400640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":489,"num_bytes_recovered":32047104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-1937}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":51,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.662515+0000","last_change":"2024-09-21T10:22:01.321467+0000","last_active":"2024-09-21T10:23:26.662515+0000","last_peered":"2024-09-21T10:23:26.662515+0000","last_clean":"2024-09-21T10:23:26.662515+0000","last_became_active":"2024-09-21T10:22:01.318270+0000","last_became_peered":"2024-09-21T10:22:01.318270+0000","last_unstale":"2024-09-21T10:23:26.662515+0000","last_undegraded":"2024-09-21T10:23:26.662515+0000","last_fullsized":"2024-09-21T10:23:26.662515+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:44.059553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":69,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.669785+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:23:26.669785+0000","last_peered":"2024-09-21T10:23:26.669785+0000","last_clean":"2024-09-21T10:23:26.669785+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:23:26.669785+0000","last_undegraded":"2024-09-21T10:23:26.669785+0000","last_fullsized":"2024-09-21T10:23:26.669785+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"362'11108","reported_seq":14986,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.980032+0000","last_change":"2024-09-21T10:20:04.684629+0000","last_active":"2024-09-21T10:23:34.980032+0000","last_peered":"2024-09-21T10:23:34.980032+0000","last_clean":"2024-09-21T10:23:34.980032+0000","last_became_active":"2024-09-21T10:20:04.680299+0000","last_became_peered":"2024-09-21T10:20:04.680299+0000","last_unstale":"2024-09-21T10:23:34.980032+0000","last_undegraded":"2024-09-21T10:23:34.980032+0000","last_fullsized":"2024-09-21T10:23:34.980032+0000","mapping_epoch":320,"log_start":"328'4500","ondisk_log_start":"328'4500","created":305,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6608,"log_dups_size":0,"ondisk_log_size":6608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:41.743081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74842112,"num_objects":1142,"num_object_clones":0,"num_object_copies":3426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11108,"num_write_kb":392000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1891,"num_bytes_recovered":123928576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":38,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.663659+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:23:26.663659+0000","last_peered":"2024-09-21T10:23:26.663659+0000","last_clean":"2024-09-21T10:23:26.663659+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:23:26.663659+0000","last_undegraded":"2024-09-21T10:23:26.663659+0000","last_fullsized":"2024-09-21T10:23:26.663659+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":58,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.671431+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:23:26.671431+0000","last_peered":"2024-09-21T10:23:26.671431+0000","last_clean":"2024-09-21T10:23:26.671431+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:23:26.671431+0000","last_undegraded":"2024-09-21T10:23:26.671431+0000","last_fullsized":"2024-09-21T10:23:26.671431+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"362'11154","reported_seq":11280,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:36.642330+0000","last_change":"2024-09-21T10:19:22.649712+0000","last_active":"2024-09-21T10:23:36.642330+0000","last_peered":"2024-09-21T10:23:36.642330+0000","last_clean":"2024-09-21T10:23:36.642330+0000","last_became_active":"2024-09-21T10:19:22.649303+0000","last_became_peered":"2024-09-21T10:19:22.649303+0000","last_unstale":"2024-09-21T10:23:36.642330+0000","last_undegraded":"2024-09-21T10:23:36.642330+0000","last_fullsized":"2024-09-21T10:23:36.642330+0000","mapping_epoch":305,"log_start":"327'3961","ondisk_log_start":"327'3961","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7193,"log_dups_size":0,"ondisk_log_size":7193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:24.442851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75366400,"num_objects":1150,"num_object_clones":0,"num_object_copies":3450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11154,"num_write_kb":393728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":360,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.662564+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:23:26.662564+0000","last_peered":"2024-09-21T10:23:26.662564+0000","last_clean":"2024-09-21T10:23:26.662564+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:23:26.662564+0000","last_undegraded":"2024-09-21T10:23:26.662564+0000","last_fullsized":"2024-09-21T10:23:26.662564+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":380,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.664314+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:23:26.664314+0000","last_peered":"2024-09-21T10:23:26.664314+0000","last_clean":"2024-09-21T10:23:26.664314+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:23:26.664314+0000","last_undegraded":"2024-09-21T10:23:26.664314+0000","last_fullsized":"2024-09-21T10:23:26.664314+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"362'11645","reported_seq":11771,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.386328+0000","last_change":"2024-09-21T10:19:22.640645+0000","last_active":"2024-09-21T10:23:35.386328+0000","last_peered":"2024-09-21T10:23:35.386328+0000","last_clean":"2024-09-21T10:23:35.386328+0000","last_became_active":"2024-09-21T10:19:22.640343+0000","last_became_peered":"2024-09-21T10:19:22.640343+0000","last_unstale":"2024-09-21T10:23:35.386328+0000","last_undegraded":"2024-09-21T10:23:35.386328+0000","last_fullsized":"2024-09-21T10:23:35.386328+0000","mapping_epoch":305,"log_start":"335'6000","ondisk_log_start":"335'6000","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:48.734122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78446592,"num_objects":1197,"num_object_clones":0,"num_object_copies":3591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11645,"num_write_kb":410944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":294,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.668979+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:23:26.668979+0000","last_peered":"2024-09-21T10:23:26.668979+0000","last_clean":"2024-09-21T10:23:26.668979+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:23:26.668979+0000","last_undegraded":"2024-09-21T10:23:26.668979+0000","last_fullsized":"2024-09-21T10:23:26.668979+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":213,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712599+0000","last_change":"2024-09-21T10:19:50.585724+0000","last_active":"2024-09-21T10:23:27.712599+0000","last_peered":"2024-09-21T10:23:27.712599+0000","last_clean":"2024-09-21T10:23:27.712599+0000","last_became_active":"2024-09-21T10:19:50.585315+0000","last_became_peered":"2024-09-21T10:19:50.585315+0000","last_unstale":"2024-09-21T10:23:27.712599+0000","last_undegraded":"2024-09-21T10:23:27.712599+0000","last_fullsized":"2024-09-21T10:23:27.712599+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:26.618970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"362'11248","reported_seq":16338,"reported_epoch":362,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:33.874738+0000","last_change":"2024-09-21T10:22:01.328242+0000","last_active":"2024-09-21T10:23:33.874738+0000","last_peered":"2024-09-21T10:23:33.874738+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492359+0000","last_became_peered":"2024-09-21T10:20:14.492359+0000","last_unstale":"2024-09-21T10:23:33.874738+0000","last_undegraded":"2024-09-21T10:20:14.218670+0000","last_fullsized":"2024-09-21T10:20:14.071510+0000","mapping_epoch":325,"log_start":"327'4400","ondisk_log_start":"327'4400","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6848,"log_dups_size":0,"ondisk_log_size":6848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83623936,"num_objects":1276,"num_object_clones":0,"num_object_copies":3828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11248,"num_write_kb":400768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2481,"num_bytes_recovered":148307968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":72}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.661194+0000","last_change":"2024-09-21T10:22:04.029688+0000","last_active":"2024-09-21T10:23:26.661194+0000","last_peered":"2024-09-21T10:23:26.661194+0000","last_clean":"2024-09-21T10:23:26.661194+0000","last_became_active":"2024-09-21T10:22:04.028853+0000","last_became_peered":"2024-09-21T10:22:04.028853+0000","last_unstale":"2024-09-21T10:23:26.661194+0000","last_undegraded":"2024-09-21T10:23:26.661194+0000","last_fullsized":"2024-09-21T10:23:26.661194+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:20.457596+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:19:20.457596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":651,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.658362+0000","last_change":"2024-09-21T10:22:01.335303+0000","last_active":"2024-09-21T10:23:27.658362+0000","last_peered":"2024-09-21T10:23:27.658362+0000","last_clean":"2024-09-21T10:23:27.658362+0000","last_became_active":"2024-09-21T10:22:01.325298+0000","last_became_peered":"2024-09-21T10:22:01.325298+0000","last_unstale":"2024-09-21T10:23:27.658362+0000","last_undegraded":"2024-09-21T10:23:27.658362+0000","last_fullsized":"2024-09-21T10:23:27.658362+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:01.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.671031+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:23:26.671031+0000","last_peered":"2024-09-21T10:23:26.671031+0000","last_clean":"2024-09-21T10:23:26.671031+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:23:26.671031+0000","last_undegraded":"2024-09-21T10:23:26.671031+0000","last_fullsized":"2024-09-21T10:23:26.671031+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"362'11106","reported_seq":11233,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.450290+0000","last_change":"2024-09-21T10:19:22.641281+0000","last_active":"2024-09-21T10:23:35.450290+0000","last_peered":"2024-09-21T10:23:35.450290+0000","last_clean":"2024-09-21T10:23:35.450290+0000","last_became_active":"2024-09-21T10:19:22.640988+0000","last_became_peered":"2024-09-21T10:19:22.640988+0000","last_unstale":"2024-09-21T10:23:35.450290+0000","last_undegraded":"2024-09-21T10:23:35.450290+0000","last_fullsized":"2024-09-21T10:23:35.450290+0000","mapping_epoch":305,"log_start":"334'5500","ondisk_log_start":"334'5500","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5606,"log_dups_size":0,"ondisk_log_size":5606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:52.652565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71827456,"num_objects":1096,"num_object_clones":0,"num_object_copies":3288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11106,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":430,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:26.661422+0000","last_change":"2024-09-21T10:22:02.013601+0000","last_active":"2024-09-21T10:23:26.661422+0000","last_peered":"2024-09-21T10:23:26.661422+0000","last_clean":"2024-09-21T10:19:15.364615+0000","last_became_active":"2024-09-21T10:22:02.013140+0000","last_became_peered":"2024-09-21T10:22:02.013140+0000","last_unstale":"2024-09-21T10:23:26.661422+0000","last_undegraded":"2024-09-21T10:22:02.004849+0000","last_fullsized":"2024-09-21T10:22:02.002552+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_clean_scrub_stamp":"2024-09-21T10:19:15.365774+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1},{"shards":"6,7","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":370,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.664429+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:23:26.664429+0000","last_peered":"2024-09-21T10:23:26.664429+0000","last_clean":"2024-09-21T10:23:26.664429+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:23:26.664429+0000","last_undegraded":"2024-09-21T10:23:26.664429+0000","last_fullsized":"2024-09-21T10:23:26.664429+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":58,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.672236+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:23:26.672236+0000","last_peered":"2024-09-21T10:23:26.672236+0000","last_clean":"2024-09-21T10:23:26.672236+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:23:26.672236+0000","last_undegraded":"2024-09-21T10:23:26.672236+0000","last_fullsized":"2024-09-21T10:23:26.672236+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"362'11405","reported_seq":18735,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.064685+0000","last_change":"2024-09-21T10:22:34.604937+0000","last_active":"2024-09-21T10:23:35.064685+0000","last_peered":"2024-09-21T10:23:35.064685+0000","last_clean":"2024-09-21T10:23:35.064685+0000","last_became_active":"2024-09-21T10:22:34.577215+0000","last_became_peered":"2024-09-21T10:22:34.577215+0000","last_unstale":"2024-09-21T10:23:35.064685+0000","last_undegraded":"2024-09-21T10:23:35.064685+0000","last_fullsized":"2024-09-21T10:23:35.064685+0000","mapping_epoch":354,"log_start":"327'4000","ondisk_log_start":"327'4000","created":305,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7405,"log_dups_size":0,"ondisk_log_size":7405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:40.287923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79233024,"num_objects":1209,"num_object_clones":0,"num_object_copies":3627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11405,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3618,"num_bytes_recovered":121634816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":58,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.671299+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:23:26.671299+0000","last_peered":"2024-09-21T10:23:26.671299+0000","last_clean":"2024-09-21T10:23:26.671299+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:23:26.671299+0000","last_undegraded":"2024-09-21T10:23:26.671299+0000","last_fullsized":"2024-09-21T10:23:26.671299+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.672900+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:23:26.672900+0000","last_peered":"2024-09-21T10:23:26.672900+0000","last_clean":"2024-09-21T10:23:26.672900+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:23:26.672900+0000","last_undegraded":"2024-09-21T10:23:26.672900+0000","last_fullsized":"2024-09-21T10:23:26.672900+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"362'11357","reported_seq":11500,"reported_epoch":362,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:35.437084+0000","last_change":"2024-09-21T10:23:24.756530+0000","last_active":"2024-09-21T10:23:35.437084+0000","last_peered":"2024-09-21T10:23:35.437084+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492538+0000","last_became_peered":"2024-09-21T10:20:14.492538+0000","last_unstale":"2024-09-21T10:23:35.437084+0000","last_undegraded":"2024-09-21T10:20:14.216051+0000","last_fullsized":"2024-09-21T10:20:14.075057+0000","mapping_epoch":325,"log_start":"335'5800","ondisk_log_start":"335'5800","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5557,"log_dups_size":0,"ondisk_log_size":5557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76349440,"num_objects":1165,"num_object_clones":0,"num_object_copies":3495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11357,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-2410}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.661323+0000","last_change":"2024-09-21T10:22:04.031320+0000","last_active":"2024-09-21T10:23:26.661323+0000","last_peered":"2024-09-21T10:23:26.661323+0000","last_clean":"2024-09-21T10:23:26.661323+0000","last_became_active":"2024-09-21T10:22:04.030774+0000","last_became_peered":"2024-09-21T10:22:04.030774+0000","last_unstale":"2024-09-21T10:23:26.661323+0000","last_undegraded":"2024-09-21T10:23:26.661323+0000","last_fullsized":"2024-09-21T10:23:26.661323+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:22.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:22.489745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":630,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.657986+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:23:27.657986+0000","last_peered":"2024-09-21T10:23:27.657986+0000","last_clean":"2024-09-21T10:23:27.657986+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:23:27.657986+0000","last_undegraded":"2024-09-21T10:23:27.657986+0000","last_fullsized":"2024-09-21T10:23:27.657986+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":322,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.710829+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:23:27.710829+0000","last_peered":"2024-09-21T10:23:27.710829+0000","last_clean":"2024-09-21T10:23:27.710829+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:23:27.710829+0000","last_undegraded":"2024-09-21T10:23:27.710829+0000","last_fullsized":"2024-09-21T10:23:27.710829+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"362'11317","reported_seq":16633,"reported_epoch":362,"state":"active+recovering+remapped","last_fresh":"2024-09-21T10:23:34.984835+0000","last_change":"2024-09-21T10:23:26.133549+0000","last_active":"2024-09-21T10:23:34.984835+0000","last_peered":"2024-09-21T10:23:34.984835+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:22:02.086281+0000","last_became_peered":"2024-09-21T10:22:02.086281+0000","last_unstale":"2024-09-21T10:23:34.984835+0000","last_undegraded":"2024-09-21T10:23:34.984835+0000","last_fullsized":"2024-09-21T10:23:34.984835+0000","mapping_epoch":347,"log_start":"328'4700","ondisk_log_start":"328'4700","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:37.553227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73728000,"num_objects":1125,"num_object_clones":0,"num_object_copies":3375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11317,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2616,"num_bytes_recovered":123076608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,5,6","objects":1125}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.661296+0000","last_change":"2024-09-21T10:22:04.029284+0000","last_active":"2024-09-21T10:23:26.661296+0000","last_peered":"2024-09-21T10:23:26.661296+0000","last_clean":"2024-09-21T10:23:26.661296+0000","last_became_active":"2024-09-21T10:22:04.028938+0000","last_became_peered":"2024-09-21T10:22:04.028938+0000","last_unstale":"2024-09-21T10:23:26.661296+0000","last_undegraded":"2024-09-21T10:23:26.661296+0000","last_fullsized":"2024-09-21T10:23:26.661296+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:12.388304+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:12.388304+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":646,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:26.662821+0000","last_change":"2024-09-21T10:21:36.010162+0000","last_active":"2024-09-21T10:23:26.662821+0000","last_peered":"2024-09-21T10:23:26.662821+0000","last_clean":"2024-09-21T10:21:17.725138+0000","last_became_active":"2024-09-21T10:21:36.009706+0000","last_became_peered":"2024-09-21T10:21:36.009706+0000","last_unstale":"2024-09-21T10:23:26.662821+0000","last_undegraded":"2024-09-21T10:21:35.852171+0000","last_fullsized":"2024-09-21T10:21:35.851526+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:04.206298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":561,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.657074+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:23:27.657074+0000","last_peered":"2024-09-21T10:23:27.657074+0000","last_clean":"2024-09-21T10:23:27.657074+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:23:27.657074+0000","last_undegraded":"2024-09-21T10:23:27.657074+0000","last_fullsized":"2024-09-21T10:23:27.657074+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":336,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.665054+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:23:26.665054+0000","last_peered":"2024-09-21T10:23:26.665054+0000","last_clean":"2024-09-21T10:23:26.665054+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:23:26.665054+0000","last_undegraded":"2024-09-21T10:23:26.665054+0000","last_fullsized":"2024-09-21T10:23:26.665054+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"362'11043","reported_seq":11169,"reported_epoch":362,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:35.422650+0000","last_change":"2024-09-21T10:20:14.480108+0000","last_active":"2024-09-21T10:23:35.422650+0000","last_peered":"2024-09-21T10:23:35.422650+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.471864+0000","last_became_peered":"2024-09-21T10:20:14.471864+0000","last_unstale":"2024-09-21T10:23:35.422650+0000","last_undegraded":"2024-09-21T10:20:14.214064+0000","last_fullsized":"2024-09-21T10:20:14.074889+0000","mapping_epoch":325,"log_start":"334'5400","ondisk_log_start":"334'5400","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5643,"log_dups_size":0,"ondisk_log_size":5643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76087296,"num_objects":1161,"num_object_clones":0,"num_object_copies":3483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11043,"num_write_kb":390528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-2384}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.661599+0000","last_change":"2024-09-21T10:22:04.031573+0000","last_active":"2024-09-21T10:23:26.661599+0000","last_peered":"2024-09-21T10:23:26.661599+0000","last_clean":"2024-09-21T10:23:26.661599+0000","last_became_active":"2024-09-21T10:22:04.031125+0000","last_became_peered":"2024-09-21T10:22:04.031125+0000","last_unstale":"2024-09-21T10:23:26.661599+0000","last_undegraded":"2024-09-21T10:23:26.661599+0000","last_fullsized":"2024-09-21T10:23:26.661599+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:19:19.450903+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1002,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.665675+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:23:26.665675+0000","last_peered":"2024-09-21T10:23:26.665675+0000","last_clean":"2024-09-21T10:23:26.665675+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:23:26.665675+0000","last_undegraded":"2024-09-21T10:23:26.665675+0000","last_fullsized":"2024-09-21T10:23:26.665675+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":211,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712283+0000","last_change":"2024-09-21T10:19:51.555124+0000","last_active":"2024-09-21T10:23:27.712283+0000","last_peered":"2024-09-21T10:23:27.712283+0000","last_clean":"2024-09-21T10:23:27.712283+0000","last_became_active":"2024-09-21T10:19:51.554766+0000","last_became_peered":"2024-09-21T10:19:51.554766+0000","last_unstale":"2024-09-21T10:23:27.712283+0000","last_undegraded":"2024-09-21T10:23:27.712283+0000","last_fullsized":"2024-09-21T10:23:27.712283+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:52.832950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":42,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712424+0000","last_change":"2024-09-21T10:22:01.316140+0000","last_active":"2024-09-21T10:23:27.712424+0000","last_peered":"2024-09-21T10:23:27.712424+0000","last_clean":"2024-09-21T10:23:27.712424+0000","last_became_active":"2024-09-21T10:22:01.315408+0000","last_became_peered":"2024-09-21T10:22:01.315408+0000","last_unstale":"2024-09-21T10:23:27.712424+0000","last_undegraded":"2024-09-21T10:23:27.712424+0000","last_fullsized":"2024-09-21T10:23:27.712424+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:07.322279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"362'11327","reported_seq":15314,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:33.811868+0000","last_change":"2024-09-21T10:20:47.589372+0000","last_active":"2024-09-21T10:23:33.811868+0000","last_peered":"2024-09-21T10:23:33.811868+0000","last_clean":"2024-09-21T10:23:33.811868+0000","last_became_active":"2024-09-21T10:20:47.580051+0000","last_became_peered":"2024-09-21T10:20:47.580051+0000","last_unstale":"2024-09-21T10:23:33.811868+0000","last_undegraded":"2024-09-21T10:23:33.811868+0000","last_fullsized":"2024-09-21T10:23:33.811868+0000","mapping_epoch":332,"log_start":"327'4500","ondisk_log_start":"327'4500","created":305,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6827,"log_dups_size":0,"ondisk_log_size":6827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:46.363318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11327,"num_write_kb":403136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.661239+0000","last_change":"2024-09-21T10:22:04.030341+0000","last_active":"2024-09-21T10:23:26.661239+0000","last_peered":"2024-09-21T10:23:26.661239+0000","last_clean":"2024-09-21T10:23:26.661239+0000","last_became_active":"2024-09-21T10:22:04.030000+0000","last_became_peered":"2024-09-21T10:22:04.030000+0000","last_unstale":"2024-09-21T10:23:26.661239+0000","last_undegraded":"2024-09-21T10:23:26.661239+0000","last_fullsized":"2024-09-21T10:23:26.661239+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:10.332770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:10.332770+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1150,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.709228+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:23:27.709228+0000","last_peered":"2024-09-21T10:23:27.709228+0000","last_clean":"2024-09-21T10:23:27.709228+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:23:27.709228+0000","last_undegraded":"2024-09-21T10:23:27.709228+0000","last_fullsized":"2024-09-21T10:23:27.709228+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":309,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.711726+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:23:27.711726+0000","last_peered":"2024-09-21T10:23:27.711726+0000","last_clean":"2024-09-21T10:23:27.711726+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:23:27.711726+0000","last_undegraded":"2024-09-21T10:23:27.711726+0000","last_fullsized":"2024-09-21T10:23:27.711726+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":338,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.711677+0000","last_change":"2024-09-21T10:22:01.317933+0000","last_active":"2024-09-21T10:23:27.711677+0000","last_peered":"2024-09-21T10:23:27.711677+0000","last_clean":"2024-09-21T10:23:27.711677+0000","last_became_active":"2024-09-21T10:22:01.317536+0000","last_became_peered":"2024-09-21T10:22:01.317536+0000","last_unstale":"2024-09-21T10:23:27.711677+0000","last_undegraded":"2024-09-21T10:23:27.711677+0000","last_fullsized":"2024-09-21T10:23:27.711677+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:34.312724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"362'11457","reported_seq":18872,"reported_epoch":362,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:23:35.056131+0000","last_change":"2024-09-21T10:23:27.721484+0000","last_active":"2024-09-21T10:23:35.056131+0000","last_peered":"2024-09-21T10:23:35.056131+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:23:27.674984+0000","last_became_peered":"2024-09-21T10:23:27.674984+0000","last_unstale":"2024-09-21T10:23:35.056131+0000","last_undegraded":"2024-09-21T10:23:35.056131+0000","last_fullsized":"2024-09-21T10:23:35.056131+0000","mapping_epoch":361,"log_start":"327'4200","ondisk_log_start":"327'4200","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7257,"log_dups_size":0,"ondisk_log_size":7257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:13.754952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79495168,"num_objects":1213,"num_object_clones":0,"num_object_copies":3639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11457,"num_write_kb":405440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3641,"num_bytes_recovered":232128512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1213}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.661285+0000","last_change":"2024-09-21T10:22:04.029892+0000","last_active":"2024-09-21T10:23:26.661285+0000","last_peered":"2024-09-21T10:23:26.661285+0000","last_clean":"2024-09-21T10:23:26.661285+0000","last_became_active":"2024-09-21T10:22:04.029353+0000","last_became_peered":"2024-09-21T10:22:04.029353+0000","last_unstale":"2024-09-21T10:23:26.661285+0000","last_undegraded":"2024-09-21T10:23:26.661285+0000","last_fullsized":"2024-09-21T10:23:26.661285+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:19:13.371825+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1104,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.663389+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:23:26.663389+0000","last_peered":"2024-09-21T10:23:26.663389+0000","last_clean":"2024-09-21T10:23:26.663389+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:23:26.663389+0000","last_undegraded":"2024-09-21T10:23:26.663389+0000","last_fullsized":"2024-09-21T10:23:26.663389+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":58,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.674680+0000","last_change":"2024-09-21T10:19:50.581529+0000","last_active":"2024-09-21T10:23:26.674680+0000","last_peered":"2024-09-21T10:23:26.674680+0000","last_clean":"2024-09-21T10:23:26.674680+0000","last_became_active":"2024-09-21T10:19:50.581119+0000","last_became_peered":"2024-09-21T10:19:50.581119+0000","last_unstale":"2024-09-21T10:23:26.674680+0000","last_undegraded":"2024-09-21T10:23:26.674680+0000","last_fullsized":"2024-09-21T10:23:26.674680+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:13.334483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":309,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.709365+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:23:27.709365+0000","last_peered":"2024-09-21T10:23:27.709365+0000","last_clean":"2024-09-21T10:23:27.709365+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:23:27.709365+0000","last_undegraded":"2024-09-21T10:23:27.709365+0000","last_fullsized":"2024-09-21T10:23:27.709365+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"362'11577","reported_seq":15692,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:36.661129+0000","last_change":"2024-09-21T10:20:42.601607+0000","last_active":"2024-09-21T10:23:36.661129+0000","last_peered":"2024-09-21T10:23:36.661129+0000","last_clean":"2024-09-21T10:23:36.661129+0000","last_became_active":"2024-09-21T10:20:42.593406+0000","last_became_peered":"2024-09-21T10:20:42.593406+0000","last_unstale":"2024-09-21T10:23:36.661129+0000","last_undegraded":"2024-09-21T10:23:36.661129+0000","last_fullsized":"2024-09-21T10:23:36.661129+0000","mapping_epoch":330,"log_start":"327'4368","ondisk_log_start":"327'4368","created":305,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7209,"log_dups_size":0,"ondisk_log_size":7209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:09.770489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75038720,"num_objects":1145,"num_object_clones":0,"num_object_copies":3435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11577,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1985,"num_bytes_recovered":130088960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.662691+0000","last_change":"2024-09-21T10:22:04.029371+0000","last_active":"2024-09-21T10:23:26.662691+0000","last_peered":"2024-09-21T10:23:26.662691+0000","last_clean":"2024-09-21T10:23:26.662691+0000","last_became_active":"2024-09-21T10:22:04.029014+0000","last_became_peered":"2024-09-21T10:22:04.029014+0000","last_unstale":"2024-09-21T10:23:26.662691+0000","last_undegraded":"2024-09-21T10:23:26.662691+0000","last_fullsized":"2024-09-21T10:23:26.662691+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":309,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.665481+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:23:26.665481+0000","last_peered":"2024-09-21T10:23:26.665481+0000","last_clean":"2024-09-21T10:23:26.665481+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:23:26.665481+0000","last_undegraded":"2024-09-21T10:23:26.665481+0000","last_fullsized":"2024-09-21T10:23:26.665481+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":917,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.709494+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:23:27.709494+0000","last_peered":"2024-09-21T10:23:27.709494+0000","last_clean":"2024-09-21T10:23:27.709494+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:23:27.709494+0000","last_undegraded":"2024-09-21T10:23:27.709494+0000","last_fullsized":"2024-09-21T10:23:27.709494+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":345,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.711386+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:23:27.711386+0000","last_peered":"2024-09-21T10:23:27.711386+0000","last_clean":"2024-09-21T10:23:27.711386+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:23:27.711386+0000","last_undegraded":"2024-09-21T10:23:27.711386+0000","last_fullsized":"2024-09-21T10:23:27.711386+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"362'11135","reported_seq":11261,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:33.869609+0000","last_change":"2024-09-21T10:19:22.641043+0000","last_active":"2024-09-21T10:23:33.869609+0000","last_peered":"2024-09-21T10:23:33.869609+0000","last_clean":"2024-09-21T10:23:33.869609+0000","last_became_active":"2024-09-21T10:19:22.640742+0000","last_became_peered":"2024-09-21T10:19:22.640742+0000","last_unstale":"2024-09-21T10:23:33.869609+0000","last_undegraded":"2024-09-21T10:23:33.869609+0000","last_fullsized":"2024-09-21T10:23:33.869609+0000","mapping_epoch":305,"log_start":"327'4300","ondisk_log_start":"327'4300","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:36.901208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79364096,"num_objects":1211,"num_object_clones":0,"num_object_copies":3633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11135,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.663731+0000","last_change":"2024-09-21T10:22:04.031872+0000","last_active":"2024-09-21T10:23:26.663731+0000","last_peered":"2024-09-21T10:23:26.663731+0000","last_clean":"2024-09-21T10:23:26.663731+0000","last_became_active":"2024-09-21T10:22:04.031343+0000","last_became_peered":"2024-09-21T10:22:04.031343+0000","last_unstale":"2024-09-21T10:23:26.663731+0000","last_undegraded":"2024-09-21T10:23:26.663731+0000","last_fullsized":"2024-09-21T10:23:26.663731+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:07.332404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:07.332404+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":327,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.709625+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:23:27.709625+0000","last_peered":"2024-09-21T10:23:27.709625+0000","last_clean":"2024-09-21T10:23:27.709625+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:23:27.709625+0000","last_undegraded":"2024-09-21T10:23:27.709625+0000","last_fullsized":"2024-09-21T10:23:27.709625+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":928,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.662688+0000","last_change":"2024-09-21T10:22:01.336961+0000","last_active":"2024-09-21T10:23:26.662688+0000","last_peered":"2024-09-21T10:23:26.662688+0000","last_clean":"2024-09-21T10:23:26.662688+0000","last_became_active":"2024-09-21T10:22:01.320079+0000","last_became_peered":"2024-09-21T10:22:01.320079+0000","last_unstale":"2024-09-21T10:23:26.662688+0000","last_undegraded":"2024-09-21T10:23:26.662688+0000","last_fullsized":"2024-09-21T10:23:26.662688+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:54.037388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":210,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712700+0000","last_change":"2024-09-21T10:20:13.088557+0000","last_active":"2024-09-21T10:23:27.712700+0000","last_peered":"2024-09-21T10:23:27.712700+0000","last_clean":"2024-09-21T10:23:27.712700+0000","last_became_active":"2024-09-21T10:20:13.088123+0000","last_became_peered":"2024-09-21T10:20:13.088123+0000","last_unstale":"2024-09-21T10:23:27.712700+0000","last_undegraded":"2024-09-21T10:23:27.712700+0000","last_fullsized":"2024-09-21T10:23:27.712700+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:49.475499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"362'11326","reported_seq":16073,"reported_epoch":362,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:23:33.896785+0000","last_change":"2024-09-21T10:22:01.405956+0000","last_active":"2024-09-21T10:23:33.896785+0000","last_peered":"2024-09-21T10:23:33.896785+0000","last_clean":"2024-09-21T10:20:12.067444+0000","last_became_active":"2024-09-21T10:22:01.391808+0000","last_became_peered":"2024-09-21T10:22:01.391808+0000","last_unstale":"2024-09-21T10:23:33.896785+0000","last_undegraded":"2024-09-21T10:22:01.123056+0000","last_fullsized":"2024-09-21T10:23:33.896785+0000","mapping_epoch":346,"log_start":"327'4500","ondisk_log_start":"327'4500","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6826,"log_dups_size":0,"ondisk_log_size":6826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80609280,"num_objects":1230,"num_object_clones":0,"num_object_copies":3690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1230,"num_objects_misplaced":1230,"num_objects_unfound":0,"num_objects_dirty":1230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11326,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2300,"num_bytes_recovered":150601728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[3,5,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,5","objects":1230}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.662346+0000","last_change":"2024-09-21T10:22:04.032207+0000","last_active":"2024-09-21T10:23:26.662346+0000","last_peered":"2024-09-21T10:23:26.662346+0000","last_clean":"2024-09-21T10:23:26.662346+0000","last_became_active":"2024-09-21T10:22:04.031881+0000","last_became_peered":"2024-09-21T10:22:04.031881+0000","last_unstale":"2024-09-21T10:23:26.662346+0000","last_undegraded":"2024-09-21T10:23:26.662346+0000","last_fullsized":"2024-09-21T10:23:26.662346+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:08.332410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:08.332410+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":635,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.658909+0000","last_change":"2024-09-21T10:21:34.876196+0000","last_active":"2024-09-21T10:23:27.658909+0000","last_peered":"2024-09-21T10:23:27.658909+0000","last_clean":"2024-09-21T10:23:27.658909+0000","last_became_active":"2024-09-21T10:21:34.868031+0000","last_became_peered":"2024-09-21T10:21:34.868031+0000","last_unstale":"2024-09-21T10:23:27.658909+0000","last_undegraded":"2024-09-21T10:23:27.658909+0000","last_fullsized":"2024-09-21T10:23:27.658909+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:58.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":980,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.710424+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:23:27.710424+0000","last_peered":"2024-09-21T10:23:27.710424+0000","last_clean":"2024-09-21T10:23:27.710424+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:23:27.710424+0000","last_undegraded":"2024-09-21T10:23:27.710424+0000","last_fullsized":"2024-09-21T10:23:27.710424+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":630,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.657190+0000","last_change":"2024-09-21T10:22:01.329400+0000","last_active":"2024-09-21T10:23:27.657190+0000","last_peered":"2024-09-21T10:23:27.657190+0000","last_clean":"2024-09-21T10:23:27.657190+0000","last_became_active":"2024-09-21T10:22:01.328887+0000","last_became_peered":"2024-09-21T10:22:01.328887+0000","last_unstale":"2024-09-21T10:23:27.657190+0000","last_undegraded":"2024-09-21T10:23:27.657190+0000","last_fullsized":"2024-09-21T10:23:27.657190+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:09.626637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"362'11217","reported_seq":20218,"reported_epoch":362,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:23:35.037171+0000","last_change":"2024-09-21T10:23:27.706027+0000","last_active":"2024-09-21T10:23:35.037171+0000","last_peered":"2024-09-21T10:23:35.037171+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:21:39.424206+0000","last_became_peered":"2024-09-21T10:21:39.424206+0000","last_unstale":"2024-09-21T10:23:35.037171+0000","last_undegraded":"2024-09-21T10:23:35.037171+0000","last_fullsized":"2024-09-21T10:23:35.037171+0000","mapping_epoch":339,"log_start":"314'1200","ondisk_log_start":"314'1200","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":10017,"log_dups_size":0,"ondisk_log_size":10017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:54.426685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75497496,"num_objects":1153,"num_object_clones":0,"num_object_copies":3459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":562,"num_objects_unfound":0,"num_objects_dirty":1153,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11217,"num_write_kb":395777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4443,"num_bytes_recovered":196083712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,2,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5,6","objects":1153}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":605,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.669979+0000","last_change":"2024-09-21T10:23:26.669475+0000","last_active":"2024-09-21T10:23:26.669979+0000","last_peered":"2024-09-21T10:23:26.669979+0000","last_clean":"2024-09-21T10:23:26.669979+0000","last_became_active":"2024-09-21T10:23:26.668318+0000","last_became_peered":"2024-09-21T10:23:26.668318+0000","last_unstale":"2024-09-21T10:23:26.669979+0000","last_undegraded":"2024-09-21T10:23:26.669979+0000","last_fullsized":"2024-09-21T10:23:26.669979+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:02.435487+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:02.435487+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":596029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":46,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712476+0000","last_change":"2024-09-21T10:23:24.764361+0000","last_active":"2024-09-21T10:23:27.712476+0000","last_peered":"2024-09-21T10:23:27.712476+0000","last_clean":"2024-09-21T10:23:27.712476+0000","last_became_active":"2024-09-21T10:22:01.317312+0000","last_became_peered":"2024-09-21T10:22:01.317312+0000","last_unstale":"2024-09-21T10:23:27.712476+0000","last_undegraded":"2024-09-21T10:23:27.712476+0000","last_fullsized":"2024-09-21T10:23:27.712476+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:32:39.440581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":647,"reported_epoch":361,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:23:26.664150+0000","last_change":"2024-09-21T10:21:34.866048+0000","last_active":"2024-09-21T10:23:26.664150+0000","last_peered":"2024-09-21T10:23:26.664150+0000","last_clean":"2024-09-21T10:21:17.726080+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:23:26.664150+0000","last_undegraded":"2024-09-21T10:21:34.856315+0000","last_fullsized":"2024-09-21T10:23:26.664150+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":334,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.671600+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:23:26.671600+0000","last_peered":"2024-09-21T10:23:26.671600+0000","last_clean":"2024-09-21T10:23:26.671600+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:23:26.671600+0000","last_undegraded":"2024-09-21T10:23:26.671600+0000","last_fullsized":"2024-09-21T10:23:26.671600+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":359,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.674745+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:23:26.674745+0000","last_peered":"2024-09-21T10:23:26.674745+0000","last_clean":"2024-09-21T10:23:26.674745+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:23:26.674745+0000","last_undegraded":"2024-09-21T10:23:26.674745+0000","last_fullsized":"2024-09-21T10:23:26.674745+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":68,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.671120+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:23:26.671120+0000","last_peered":"2024-09-21T10:23:26.671120+0000","last_clean":"2024-09-21T10:23:26.671120+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:23:26.671120+0000","last_undegraded":"2024-09-21T10:23:26.671120+0000","last_fullsized":"2024-09-21T10:23:26.671120+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.663249+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:23:26.663249+0000","last_peered":"2024-09-21T10:23:26.663249+0000","last_clean":"2024-09-21T10:23:26.663249+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:23:26.663249+0000","last_undegraded":"2024-09-21T10:23:26.663249+0000","last_fullsized":"2024-09-21T10:23:26.663249+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":40,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.662023+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:23:26.662023+0000","last_peered":"2024-09-21T10:23:26.662023+0000","last_clean":"2024-09-21T10:23:26.662023+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:23:26.662023+0000","last_undegraded":"2024-09-21T10:23:26.662023+0000","last_fullsized":"2024-09-21T10:23:26.662023+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.662715+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:23:26.662715+0000","last_peered":"2024-09-21T10:23:26.662715+0000","last_clean":"2024-09-21T10:23:26.662715+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:23:26.662715+0000","last_undegraded":"2024-09-21T10:23:26.662715+0000","last_fullsized":"2024-09-21T10:23:26.662715+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":554,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.658283+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:23:27.658283+0000","last_peered":"2024-09-21T10:23:27.658283+0000","last_clean":"2024-09-21T10:23:27.658283+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:23:27.658283+0000","last_undegraded":"2024-09-21T10:23:27.658283+0000","last_fullsized":"2024-09-21T10:23:27.658283+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":209,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712893+0000","last_change":"2024-09-21T10:20:13.089473+0000","last_active":"2024-09-21T10:23:27.712893+0000","last_peered":"2024-09-21T10:23:27.712893+0000","last_clean":"2024-09-21T10:23:27.712893+0000","last_became_active":"2024-09-21T10:20:13.089111+0000","last_became_peered":"2024-09-21T10:20:13.089111+0000","last_unstale":"2024-09-21T10:23:27.712893+0000","last_undegraded":"2024-09-21T10:23:27.712893+0000","last_fullsized":"2024-09-21T10:23:27.712893+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:06.603284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":614,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.657082+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:23:27.657082+0000","last_peered":"2024-09-21T10:23:27.657082+0000","last_clean":"2024-09-21T10:23:27.657082+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:23:27.657082+0000","last_undegraded":"2024-09-21T10:23:27.657082+0000","last_fullsized":"2024-09-21T10:23:27.657082+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":375,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.666525+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:23:26.666525+0000","last_peered":"2024-09-21T10:23:26.666525+0000","last_clean":"2024-09-21T10:23:26.666525+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:23:26.666525+0000","last_undegraded":"2024-09-21T10:23:26.666525+0000","last_fullsized":"2024-09-21T10:23:26.666525+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":631,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.657880+0000","last_change":"2024-09-21T10:22:01.329690+0000","last_active":"2024-09-21T10:23:27.657880+0000","last_peered":"2024-09-21T10:23:27.657880+0000","last_clean":"2024-09-21T10:23:27.657880+0000","last_became_active":"2024-09-21T10:22:01.329033+0000","last_became_peered":"2024-09-21T10:22:01.329033+0000","last_unstale":"2024-09-21T10:23:27.657880+0000","last_undegraded":"2024-09-21T10:23:27.657880+0000","last_fullsized":"2024-09-21T10:23:27.657880+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:24.559524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":213,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712661+0000","last_change":"2024-09-21T10:21:34.870704+0000","last_active":"2024-09-21T10:23:27.712661+0000","last_peered":"2024-09-21T10:23:27.712661+0000","last_clean":"2024-09-21T10:23:27.712661+0000","last_became_active":"2024-09-21T10:21:34.869835+0000","last_became_peered":"2024-09-21T10:21:34.869835+0000","last_unstale":"2024-09-21T10:23:27.712661+0000","last_undegraded":"2024-09-21T10:23:27.712661+0000","last_fullsized":"2024-09-21T10:23:27.712661+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:34.959915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":329,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.709221+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:23:27.709221+0000","last_peered":"2024-09-21T10:23:27.709221+0000","last_clean":"2024-09-21T10:23:27.709221+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:23:27.709221+0000","last_undegraded":"2024-09-21T10:23:27.709221+0000","last_fullsized":"2024-09-21T10:23:27.709221+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":620,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.657482+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:23:27.657482+0000","last_peered":"2024-09-21T10:23:27.657482+0000","last_clean":"2024-09-21T10:23:27.657482+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:23:27.657482+0000","last_undegraded":"2024-09-21T10:23:27.657482+0000","last_fullsized":"2024-09-21T10:23:27.657482+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":366,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.672669+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:23:26.672669+0000","last_peered":"2024-09-21T10:23:26.672669+0000","last_clean":"2024-09-21T10:23:26.672669+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:23:26.672669+0000","last_undegraded":"2024-09-21T10:23:26.672669+0000","last_fullsized":"2024-09-21T10:23:26.672669+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":620,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.711044+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:23:27.711044+0000","last_peered":"2024-09-21T10:23:27.711044+0000","last_clean":"2024-09-21T10:23:27.711044+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:23:27.711044+0000","last_undegraded":"2024-09-21T10:23:27.711044+0000","last_fullsized":"2024-09-21T10:23:27.711044+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":42,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712370+0000","last_change":"2024-09-21T10:22:01.318549+0000","last_active":"2024-09-21T10:23:27.712370+0000","last_peered":"2024-09-21T10:23:27.712370+0000","last_clean":"2024-09-21T10:23:27.712370+0000","last_became_active":"2024-09-21T10:22:01.318071+0000","last_became_peered":"2024-09-21T10:22:01.318071+0000","last_unstale":"2024-09-21T10:23:27.712370+0000","last_undegraded":"2024-09-21T10:23:27.712370+0000","last_fullsized":"2024-09-21T10:23:27.712370+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:49.061492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":58,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.674555+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:23:26.674555+0000","last_peered":"2024-09-21T10:23:26.674555+0000","last_clean":"2024-09-21T10:23:26.674555+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:23:26.674555+0000","last_undegraded":"2024-09-21T10:23:26.674555+0000","last_fullsized":"2024-09-21T10:23:26.674555+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":301,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.668846+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:23:26.668846+0000","last_peered":"2024-09-21T10:23:26.668846+0000","last_clean":"2024-09-21T10:23:26.668846+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:23:26.668846+0000","last_undegraded":"2024-09-21T10:23:26.668846+0000","last_fullsized":"2024-09-21T10:23:26.668846+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":58,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.672435+0000","last_change":"2024-09-21T10:19:50.588363+0000","last_active":"2024-09-21T10:23:26.672435+0000","last_peered":"2024-09-21T10:23:26.672435+0000","last_clean":"2024-09-21T10:23:26.672435+0000","last_became_active":"2024-09-21T10:19:50.588107+0000","last_became_peered":"2024-09-21T10:19:50.588107+0000","last_unstale":"2024-09-21T10:23:26.672435+0000","last_undegraded":"2024-09-21T10:23:26.672435+0000","last_fullsized":"2024-09-21T10:23:26.672435+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:05.698018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":279,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.665555+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:23:26.665555+0000","last_peered":"2024-09-21T10:23:26.665555+0000","last_clean":"2024-09-21T10:23:26.665555+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:23:26.665555+0000","last_undegraded":"2024-09-21T10:23:26.665555+0000","last_fullsized":"2024-09-21T10:23:26.665555+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":634,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.658782+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:23:27.658782+0000","last_peered":"2024-09-21T10:23:27.658782+0000","last_clean":"2024-09-21T10:23:27.658782+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:23:27.658782+0000","last_undegraded":"2024-09-21T10:23:27.658782+0000","last_fullsized":"2024-09-21T10:23:27.658782+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":595,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.658004+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:23:27.658004+0000","last_peered":"2024-09-21T10:23:27.658004+0000","last_clean":"2024-09-21T10:23:27.658004+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:23:27.658004+0000","last_undegraded":"2024-09-21T10:23:27.658004+0000","last_fullsized":"2024-09-21T10:23:27.658004+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-21T10:23:26.671034+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:23:26.671034+0000","last_peered":"2024-09-21T10:23:26.671034+0000","last_clean":"2024-09-21T10:23:26.671034+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:23:26.671034+0000","last_undegraded":"2024-09-21T10:23:26.671034+0000","last_fullsized":"2024-09-21T10:23:26.671034+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":48,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-21T10:23:27.712831+0000","last_change":"2024-09-21T10:23:24.753979+0000","last_active":"2024-09-21T10:23:27.712831+0000","last_peered":"2024-09-21T10:23:27.712831+0000","last_clean":"2024-09-21T10:23:27.712831+0000","last_became_active":"2024-09-21T10:22:01.316569+0000","last_became_peered":"2024-09-21T10:22:01.316569+0000","last_unstale":"2024-09-21T10:23:27.712831+0000","last_undegraded":"2024-09-21T10:23:27.712831+0000","last_fullsized":"2024-09-21T10:23:27.712831+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:35:44.904752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1243414552,"num_objects":18974,"num_object_clones":0,"num_object_copies":56922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12721,"num_objects_misplaced":1792,"num_objects_unfound":0,"num_objects_dirty":18974,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":180712,"num_write_kb":6389889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25389,"num_bytes_recovered":1383989248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4500500480,"data_stored":4500488264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109483,"ondisk_log_size":109483,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":6328,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24225,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":391,"num_read_kb":833,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3434868,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1640,"ondisk_log_size":1640,"up":33,"acting":32,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234407,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":882824,"kb_used_data":284032,"kb_used_omap":1,"kb_used_meta":598782,"kb_avail":92862328,"statfs":{"total":95995035648,"available":95091023872,"internally_reserved":0,"allocated":290848768,"data_stored":288976409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":613153227},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496045,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1561488,"kb_used_data":938888,"kb_used_omap":1,"kb_used_meta":622590,"kb_avail":92183664,"statfs":{"total":95995035648,"available":94396071936,"internally_reserved":0,"allocated":961421312,"data_stored":959542115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":637532615},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403945,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":957696,"kb_used_data":691128,"kb_used_omap":7,"kb_used_meta":266552,"kb_avail":92787456,"statfs":{"total":95995035648,"available":95014354944,"internally_reserved":0,"allocated":707715072,"data_stored":705846959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":272950168},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019318,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1075144,"kb_used_data":480448,"kb_used_omap":1,"kb_used_meta":594686,"kb_avail":92670008,"statfs":{"total":95995035648,"available":94894088192,"internally_reserved":0,"allocated":491978752,"data_stored":490106654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":608958918},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435502,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1577140,"kb_used_data":797164,"kb_used_omap":5,"kb_used_meta":779962,"kb_avail":92168012,"statfs":{"total":95995035648,"available":94380044288,"internally_reserved":0,"allocated":816295936,"data_stored":814439518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":798681128},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599029,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181628,"kb_used_data":486260,"kb_used_omap":7,"kb_used_meta":695288,"kb_avail":92563524,"statfs":{"total":95995035648,"available":94785048576,"internally_reserved":0,"allocated":497930240,"data_stored":496066416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":711975373},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762556,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1143416,"kb_used_data":530032,"kb_used_omap":7,"kb_used_meta":613368,"kb_avail":92601736,"statfs":{"total":95995035648,"available":94824177664,"internally_reserved":0,"allocated":542752768,"data_stored":540892476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":628089085},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926083,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":775992,"kb_used_data":302832,"kb_used_omap":7,"kb_used_meta":473144,"kb_avail":92969160,"statfs":{"total":95995035648,"available":95200419840,"internally_reserved":0,"allocated":310099968,"data_stored":308239968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":484499646},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":292945920,"data_stored":292945920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532545536,"data_stored":532545536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":474288128,"data_stored":474284056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":809828352,"data_stored":809828352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":470876160,"data_stored":470876160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":699142144,"data_stored":699138072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":945360896,"data_stored":945356824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":275513344,"data_stored":275513344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:23:40.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:40.288 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:40.390 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:40.417 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:40.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:40.519 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:40.554 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:40.656 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:40.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:40.780 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:40.796 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:40.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:40.942 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.990 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:40.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:41.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:41.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.128 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:41.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:41.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:41.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:41.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:41.361 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:41.397 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:41.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:41.498 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:41.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:41.636 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:41.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:41.719 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:23:41.775 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:41.817 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:41.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:41.919 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:41.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.954 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:41.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:41 smithi045 ceph-mon[95976]: pgmap v1096: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 959 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 12481/45885 objects degraded (27.201%); 1394/45885 objects misplaced (3.038%); 4.1 MiB/s, 171 objects/s recovering 2024-09-21T10:23:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:41 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3374385685' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:42.055 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:42.088 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:42.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:42.190 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:42.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:41 smithi032 ceph-mon[105748]: pgmap v1096: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 959 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 12481/45885 objects degraded (27.201%); 1394/45885 objects misplaced (3.038%); 4.1 MiB/s, 171 objects/s recovering 2024-09-21T10:23:42.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:41 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3374385685' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:42.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:41 smithi032 ceph-mon[101299]: pgmap v1096: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 959 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 12481/45885 objects degraded (27.201%); 1394/45885 objects misplaced (3.038%); 4.1 MiB/s, 171 objects/s recovering 2024-09-21T10:23:42.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:41 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3374385685' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:42.213 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:42.314 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:42.354 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:42.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:42.455 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:42.477 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:42.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:42.579 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:42.612 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:42.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:42.714 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:42.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:42.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:42.851 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:42.881 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:42.983 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:43.011 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:42 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:42 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3374385685' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:42 smithi045 ceph-mon[95976]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T10:23:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:42 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12481/45885 objects degraded (27.201%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:43.074 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:42 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:43.075 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:42 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3374385685' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:43.075 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:42 smithi032 ceph-mon[105748]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T10:23:43.075 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:42 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12481/45885 objects degraded (27.201%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:43.076 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:42 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:43.076 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:42 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3374385685' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:43.076 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:42 smithi032 ceph-mon[101299]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T10:23:43.076 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:42 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12481/45885 objects degraded (27.201%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:43.112 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:43.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.150 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:43.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:43.201 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:23:43.251 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:43.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:43.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:43.385 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:43.432 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:43.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:43.534 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:43.574 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:43.678 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:43.713 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:43.815 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:43.850 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:43.952 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:43.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:43.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:43 smithi045 ceph-mon[95976]: pgmap v1098: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 925 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 12481/44250 objects degraded (28.206%); 1394/44250 objects misplaced (3.150%); 0 B/s, 80 objects/s recovering 2024-09-21T10:23:44.091 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:44.134 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:43 smithi032 ceph-mon[101299]: pgmap v1098: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 925 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 12481/44250 objects degraded (28.206%); 1394/44250 objects misplaced (3.150%); 0 B/s, 80 objects/s recovering 2024-09-21T10:23:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:43 smithi032 ceph-mon[105748]: pgmap v1098: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 925 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 12481/44250 objects degraded (28.206%); 1394/44250 objects misplaced (3.150%); 0 B/s, 80 objects/s recovering 2024-09-21T10:23:44.236 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:44.273 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:44.375 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:44.417 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:44.519 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:44.552 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:44.654 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:44.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:44.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:44.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:44.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.830 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:44.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:44.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:44.974 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:44.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:45.076 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:45.106 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:45.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:45.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:45.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.259 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:45.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:45.360 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:45.397 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:45.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:45.499 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:45.530 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:45.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:45.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:45.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:45.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:45.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:45.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.819 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:45.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:45.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:45.953 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:45.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:46.013 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:45 smithi032 ceph-mon[101299]: pgmap v1099: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 860 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 12229/41160 objects degraded (29.711%); 1142/41160 objects misplaced (2.775%); 0 B/s, 80 objects/s recovering 2024-09-21T10:23:46.013 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:45 smithi032 ceph-mon[105748]: pgmap v1099: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 860 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 12229/41160 objects degraded (29.711%); 1142/41160 objects misplaced (2.775%); 0 B/s, 80 objects/s recovering 2024-09-21T10:23:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:45 smithi045 ceph-mon[95976]: pgmap v1099: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 860 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 12229/41160 objects degraded (29.711%); 1142/41160 objects misplaced (2.775%); 0 B/s, 80 objects/s recovering 2024-09-21T10:23:46.055 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:46.105 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:46.207 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:46.237 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:46.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:46.338 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:46.395 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:46.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:46.420 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:23:46.431 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:23:46.431 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:23:46.497 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:46.531 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:46.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:46.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:46.667 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:46.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:46.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:46.773 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:23:46.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:23:46.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:23:46.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:23:46.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:23:46.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:23:46.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:23:46.774 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:23:46.774 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:23:46.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:23:46.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:23:46.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:23:46.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:23:46.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:23:46.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:23:46.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:23:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.807 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:46.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:46.908 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:46.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:46.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:46.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:46 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/311397061' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:46.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:46 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/311397061' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:46 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/311397061' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:47.051 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:47.091 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:47.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:47.192 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:47.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:47.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:47.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:47.360 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:47.462 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:47.500 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:47.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:47.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:47.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:47.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:47.742 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:47.780 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:47.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:47.882 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:47.918 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:47.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:48.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:47 smithi045 ceph-mon[95976]: pgmap v1100: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 695 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 12229/33210 objects degraded (36.823%); 1011/33210 objects misplaced (3.044%); 0 B/s, 89 objects/s recovering 2024-09-21T10:23:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:47 smithi045 ceph-mon[95976]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T10:23:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:47 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12229/33210 objects degraded (36.823%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:48.054 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:48.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:48.156 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:48.171 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:47 smithi032 ceph-mon[101299]: pgmap v1100: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 695 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 12229/33210 objects degraded (36.823%); 1011/33210 objects misplaced (3.044%); 0 B/s, 89 objects/s recovering 2024-09-21T10:23:48.171 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:47 smithi032 ceph-mon[101299]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T10:23:48.171 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:47 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12229/33210 objects degraded (36.823%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:48.172 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:47 smithi032 ceph-mon[105748]: pgmap v1100: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 695 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 12229/33210 objects degraded (36.823%); 1011/33210 objects misplaced (3.044%); 0 B/s, 89 objects/s recovering 2024-09-21T10:23:48.172 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:47 smithi032 ceph-mon[105748]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T10:23:48.172 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:47 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12229/33210 objects degraded (36.823%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:48.183 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:48.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:48.285 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:48.309 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:48.411 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:48.436 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:48.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:48.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:48.560 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:48.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:48.662 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:48.684 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:48.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:48.785 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:48.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.811 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:48.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:48.913 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:48.947 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:49.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:48 smithi045 ceph-mon[95976]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T10:23:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:49.039 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:48 smithi032 ceph-mon[105748]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T10:23:49.039 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:49.040 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:48 smithi032 ceph-mon[101299]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T10:23:49.040 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:49.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:49.079 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:49.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:49.181 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.204 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:49.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:49.305 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:49.327 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:49.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:49.429 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:49.452 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:49.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:49.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:23:49.554 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:49.570 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:49.584 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:49.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:49.601 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:49.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:49.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:49.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:49.746 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:23:49.769 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:23:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:23:49] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-21T10:23:49.811 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:49.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.846 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:49.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:49.868 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:49 smithi045 ceph-mon[95976]: pgmap v1103: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 661 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 12229/31605 objects degraded (38.693%); 1011/31605 objects misplaced (3.199%); 0 B/s, 66 objects/s recovering 2024-09-21T10:23:49.868 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:49 smithi045 ceph-mon[95976]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T10:23:49.947 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:49.988 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:50.091 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:50.125 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:49 smithi032 ceph-mon[105748]: pgmap v1103: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 661 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 12229/31605 objects degraded (38.693%); 1011/31605 objects misplaced (3.199%); 0 B/s, 66 objects/s recovering 2024-09-21T10:23:50.125 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:49 smithi032 ceph-mon[105748]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T10:23:50.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:49 smithi032 ceph-mon[101299]: pgmap v1103: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 661 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 12229/31605 objects degraded (38.693%); 1011/31605 objects misplaced (3.199%); 0 B/s, 66 objects/s recovering 2024-09-21T10:23:50.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:49 smithi032 ceph-mon[101299]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T10:23:50.132 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:50.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:50.234 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:50.277 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:50.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:50.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:50.408 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:50.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:50.532 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:50.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:50.669 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:50.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:50.771 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:50.809 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:50.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:50.910 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.947 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:50.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:50 smithi045 ceph-mon[95976]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T10:23:51.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:23:51.047 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:51.075 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:50 smithi032 ceph-mon[101299]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T10:23:51.075 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:50 smithi032 ceph-mon[105748]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T10:23:51.083 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:51.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:51.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:51.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:51.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:51.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:51.380 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:51.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:51.514 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:51.616 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:51.651 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:51.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:51.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:51.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:51.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:51.831 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:23:51.831 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:23:51.831 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.831 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:193604 8.e e367/330 8:746edbbc:::benchmark_data_smithi032_233595_object93759:head v 367'12310, mlcod=367'12308)", 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:51.824724+0000", 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0036061869999999998, 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003635564, 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824724+0000", 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.832 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824724+0000", 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824723+0000", 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824729+0000", 2024-09-21T10:23:51.833 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6210000000000001e-06 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824742+0000", 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3216e-05 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824748+0000", 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.648e-06 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:51.834 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824801+0000", 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3479000000000001e-05 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824819+0000", 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8260000000000001e-05 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:193612 8.e e367/330)", 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:51.826207+0000", 2024-09-21T10:23:51.835 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0021232870000000002, 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002219773, 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826207+0000", 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826207+0000", 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.836 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826206+0000", 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826213+0000", 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.323e-06 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826229+0000", 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6509999999999999e-05 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.837 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826236+0000", 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2659999999999998e-06 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.35586.0:193610 8.4 e367/325)", 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:51.826272+0000", 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0020580210000000002, 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0021967219999999999, 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:23:51.838 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826272+0000", 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826272+0000", 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826271+0000", 2024-09-21T10:23:51.839 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826276+0000", 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9749999999999998e-06 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826289+0000", 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2522e-05 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.826294+0000", 2024-09-21T10:23:51.840 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6200000000000004e-06 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.827863+0000", 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015683520000000001 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:193618 8.4 8.52855224 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:51.827657+0000", 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00067316299999999995, 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00086238999999999999, 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:51.841 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 193618 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.827657+0000", 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:51.842 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.827657+0000", 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.827655+0000", 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.827662+0000", 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3960000000000003e-06 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.843 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.844 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:51.844 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.827679+0000", 2024-09-21T10:23:51.844 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7819e-05 2024-09-21T10:23:51.844 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.844 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.844 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:51.844 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.827688+0000", 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9299999999999992e-06 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:193610 8.4 8:274879d5:::benchmark_data_smithi032_233595_object93765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:51.818123+0000", 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.010207437, 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010455712000000001, 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:51.845 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:23:51.846 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:51.846 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:51.846 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:51.846 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 193610 2024-09-21T10:23:51.846 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.846 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:51.846 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.846 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.818123+0000", 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.818123+0000", 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.818120+0000", 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.818129+0000", 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3999999999999992e-06 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.818146+0000", 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6714e-05 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.818154+0000", 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0090000000000007e-06 2024-09-21T10:23:51.848 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.818206+0000", 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2064000000000002e-05 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.818353+0000", 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014697199999999999 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:51.849 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.818511+0000", 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015818700000000001 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:193606 8.c e367/346 8:39ca37d0:::benchmark_data_smithi032_233595_object93761:head v 367'12168, mlcod=335'6278)", 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:51.820224+0000", 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0081059840000000001, 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0084221739999999993, 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:51.850 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.820224+0000", 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.820224+0000", 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.820223+0000", 2024-09-21T10:23:51.851 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.820231+0000", 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8390000000000007e-06 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.820248+0000", 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7404000000000001e-05 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.852 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.820256+0000", 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0199999999999994e-06 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.820423+0000", 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000167078 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.820483+0000", 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9493000000000001e-05 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:51.853 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.35586.0:193609 8.c e367/346 8:3c9ab6a0:::benchmark_data_smithi032_233595_object93764:head v 367'12169, mlcod=335'6278)", 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:51.824181+0000", 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0041489810000000004, 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0045194639999999999, 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824181+0000", 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.854 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824181+0000", 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824180+0000", 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:51.855 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824187+0000", 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1040000000000001e-06 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824206+0000", 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9190000000000001e-05 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.824212+0000", 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9499999999999998e-06 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.856 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.827928+0000", 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003715281 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:51.827980+0000", 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2453000000000001e-05 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 7 2024-09-21T10:23:51.857 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:23:51.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:51.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:51.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:52.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:52.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:51 smithi045 ceph-mon[95976]: pgmap v1106: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 489 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 11660/23361 objects degraded (49.912%); 500/23361 objects misplaced (2.140%); 12 MiB/s, 314 objects/s recovering 2024-09-21T10:23:52.069 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:52.171 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:52.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.206 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:52.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:51 smithi032 ceph-mon[101299]: pgmap v1106: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 489 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 11660/23361 objects degraded (49.912%); 500/23361 objects misplaced (2.140%); 12 MiB/s, 314 objects/s recovering 2024-09-21T10:23:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:51 smithi032 ceph-mon[105748]: pgmap v1106: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 489 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 11660/23361 objects degraded (49.912%); 500/23361 objects misplaced (2.140%); 12 MiB/s, 314 objects/s recovering 2024-09-21T10:23:52.308 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:52.358 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:52.459 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:52.497 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:52.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:52.572 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:23:52.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:52.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:52.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:52.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:52.766 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:52.868 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:52.895 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:52.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:52.931 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:23:52.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:52 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 11660/23361 objects degraded (49.912%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:52.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:52 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 11660/23361 objects degraded (49.912%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:52.997 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:53.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.062 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:53.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:53.078 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:52 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 11660/23361 objects degraded (49.912%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:53.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:53.208 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:53.310 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:53.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:53.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:53.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:53.500 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:53.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:53.602 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:53.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:53.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:53.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:53.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:53.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:53.888 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:53.920 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:53.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:54.022 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:54.064 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:54.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:54.127 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:53 smithi045 ceph-mon[95976]: pgmap v1107: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 465 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 11660/22191 objects degraded (52.544%); 500/22191 objects misplaced (2.253%); 8.1 MiB/s, 216 objects/s recovering 2024-09-21T10:23:54.166 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:54.206 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:53 smithi032 ceph-mon[101299]: pgmap v1107: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 465 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 11660/22191 objects degraded (52.544%); 500/22191 objects misplaced (2.253%); 8.1 MiB/s, 216 objects/s recovering 2024-09-21T10:23:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:53 smithi032 ceph-mon[105748]: pgmap v1107: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 465 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 11660/22191 objects degraded (52.544%); 500/22191 objects misplaced (2.253%); 8.1 MiB/s, 216 objects/s recovering 2024-09-21T10:23:54.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:54.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:54.357 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:54.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:54.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:54.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:54.598 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:54.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.675 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:54.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:54.776 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:54.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:54.913 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:54.946 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:54.951 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:23:54.951 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:23:54.951 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:23:54.951 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:23:54.951 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:23:55.048 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:55.093 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:55.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:55.194 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:55.244 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:55.287 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:55.346 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:55.395 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:55.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:55.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:55.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:55.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:23:55.689 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:55.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:55.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.835 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:55.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:55.904 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:55 smithi032 ceph-mon[101299]: pgmap v1108: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 410 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11439/19527 objects degraded (58.580%); 279/19527 objects misplaced (1.429%); 6.9 MiB/s, 185 objects/s recovering 2024-09-21T10:23:55.904 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:55 smithi032 ceph-mon[105748]: pgmap v1108: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 410 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11439/19527 objects degraded (58.580%); 279/19527 objects misplaced (1.429%); 6.9 MiB/s, 185 objects/s recovering 2024-09-21T10:23:55.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:55.977 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:56.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:55 smithi045 ceph-mon[95976]: pgmap v1108: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 410 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11439/19527 objects degraded (58.580%); 279/19527 objects misplaced (1.429%); 6.9 MiB/s, 185 objects/s recovering 2024-09-21T10:23:56.045 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:23:56.079 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:56.127 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:56.229 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:56.266 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:56.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:56.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:56.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:56.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:56.534 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:56.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:56.635 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:56.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:56.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:56.775 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.812 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:56.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:56.913 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:56.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.957 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:56.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:57.058 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:57.095 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:57.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:57.197 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:56 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 11439/19527 objects degraded (58.580%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:57.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:56 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 11439/19527 objects degraded (58.580%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:57.215 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:56 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 11439/19527 objects degraded (58.580%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:57.235 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:57.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:57.337 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:57.380 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:57.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:57.482 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.515 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:57.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:57.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:57.659 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:57.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:57.761 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:57.799 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:57.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:57.901 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.939 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:57.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:58.020 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:30959 8.4 8:2238d9c4:::benchmark_data_smithi032_233595_object30958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e316)", 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:19:50.548938+0000", 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 247.46671897100001, 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0154910880000001, 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.021 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 30959 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.548938+0000", 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.548938+0000", 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.548936+0000", 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.548990+0000", 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4184999999999997e-05 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549010+0000", 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9765000000000001e-05 2024-09-21T10:23:58.023 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549023+0000", 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2806000000000001e-05 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.551575+0000", 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0025527430000000001 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.551586+0000", 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0479000000000001e-05 2024-09-21T10:23:58.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.558717+0000", 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.007130766 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.558869+0000", 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015219699999999999 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.564231+0000", 2024-09-21T10:23:58.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.005362098 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.564429+0000", 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000197794 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:30965 8.2 8:4c29710f:::benchmark_data_smithi032_233595_object30964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e316)", 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:19:50.549036+0000", 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 247.46662067299999, 2024-09-21T10:23:58.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.015127304, 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 30965 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549036+0000", 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549036+0000", 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549035+0000", 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549135+0000", 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.9926000000000005e-05 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549153+0000", 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8247999999999999e-05 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549164+0000", 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.095e-05 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549282+0000", 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000117693 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549295+0000", 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2867000000000001e-05 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.555284+0000", 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0059891459999999 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.556233+0000", 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00094939000000000004 2024-09-21T10:23:58.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.556438+0000", 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000204978 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.560280+0000", 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0038418049999999998 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.563641+0000", 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0033611359999999998 2024-09-21T10:23:58.031 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.563938+0000", 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000297192 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.564053+0000", 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011416099999999999 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.564163+0000", 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000110577 2024-09-21T10:23:58.032 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:30967 8.2 8:450378e9:::benchmark_data_smithi032_233595_object30966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e316)", 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:19:50.549187+0000", 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 247.46646903999999, 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.020272579, 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.033 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 30967 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549187+0000", 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549187+0000", 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549187+0000", 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549266+0000", 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9498000000000004e-05 2024-09-21T10:23:58.035 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549281+0000", 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4882000000000001e-05 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549290+0000", 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6540000000000005e-06 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549320+0000", 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0599000000000003e-05 2024-09-21T10:23:58.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549328+0000", 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2660000000000004e-06 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.558166+0000", 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0088384989999999 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.558322+0000", 2024-09-21T10:23:58.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000155458 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.558532+0000", 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00021048599999999999 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.563441+0000", 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049091259999999998 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.566433+0000", 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029914160000000002 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.569271+0000", 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002838508 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.569337+0000", 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5982000000000006e-05 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.039 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.569460+0000", 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012286900000000001 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:30971 8.4 8:237e6e7f:::benchmark_data_smithi032_233595_object30970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e316)", 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:19:50.549302+0000", 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 247.466354112, 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0159435400000001, 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.040 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 30971 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549302+0000", 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549302+0000", 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549302+0000", 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549356+0000", 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4858999999999998e-05 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549368+0000", 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1639e-05 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549384+0000", 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6036e-05 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.551611+0000", 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022271690000000002 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.551619+0000", 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8339999999999999e-06 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.559219+0000", 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.007600104 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.559693+0000", 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00047345999999999998 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.559904+0000", 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000211394 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.564412+0000", 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0045082350000000002 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.564805+0000", 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000392958 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.565018+0000", 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000213248 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.565113+0000", 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4598000000000005e-05 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.565246+0000", 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013286 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.046 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:30991 8.2 8:418accec:::benchmark_data_smithi032_233595_object30990:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e316)", 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:19:50.549393+0000", 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 247.46626364299999, 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.021554708, 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 30991 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.047 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549393+0000", 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549393+0000", 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549392+0000", 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.048 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549436+0000", 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3566999999999998e-05 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549446+0000", 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.7739999999999999e-06 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549465+0000", 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9259e-05 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549580+0000", 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115287 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.549592+0000", 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1548e-05 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.560328+0000", 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0107361560000001 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.050 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.560503+0000", 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017535200000000001 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.560658+0000", 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015436399999999999 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.566867+0000", 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0062089700000000003 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.568718+0000", 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0018510130000000001 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.570793+0000", 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0020758170000000002 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.570849+0000", 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6109000000000001e-05 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.570948+0000", 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8114000000000003e-05 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:139693 8.2 8:45296cd2:::benchmark_data_smithi032_233595_object39848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:33.592198+0000", 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 84.423458584000002, 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.01296378, 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.053 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 139693 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.592198+0000", 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.592198+0000", 2024-09-21T10:23:58.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.592196+0000", 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.592204+0000", 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8129999999999996e-06 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.592216+0000", 2024-09-21T10:23:58.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.226e-05 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.592226+0000", 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3179999999999996e-06 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.592288+0000", 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2185000000000002e-05 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.592296+0000", 2024-09-21T10:23:58.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4719999999999995e-06 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.601075+0000", 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0087786139999999 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.601159+0000", 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4089000000000001e-05 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.601235+0000", 2024-09-21T10:23:58.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.559e-05 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.605037+0000", 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003802354 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.605074+0000", 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7378e-05 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.605093+0000", 2024-09-21T10:23:58.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8692999999999999e-05 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.605128+0000", 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4582000000000002e-05 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.605162+0000", 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4020000000000003e-05 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.060 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.060 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:139695 8.2 8:448bd2c2:::benchmark_data_smithi032_233595_object39850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-21T10:23:58.060 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:33.603935+0000", 2024-09-21T10:23:58.060 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 84.411721334999996, 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.007516302, 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 139695 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.061 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.603935+0000", 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.603935+0000", 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.603934+0000", 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.062 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.603940+0000", 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7730000000000002e-06 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.603954+0000", 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3286e-05 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.603961+0000", 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.401e-06 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.063 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.604008+0000", 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6644000000000003e-05 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.604017+0000", 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4879999999999996e-06 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.605197+0000", 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0011794869999999 2024-09-21T10:23:58.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.605285+0000", 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8826000000000006e-05 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.605356+0000", 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0906000000000002e-05 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.611340+0000", 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0059833940000000004 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.611392+0000", 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2478000000000001e-05 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.611398+0000", 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3909999999999997e-06 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.611428+0000", 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0825999999999999e-05 2024-09-21T10:23:58.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:34.611451+0000", 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3011e-05 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:192435 8.4 8:25d6fb97:::benchmark_data_smithi032_233595_object92590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:50.338061+0000", 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 7.6775951710000001, 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.22172612899999999, 2024-09-21T10:23:58.067 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 192435 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.338061+0000", 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.338061+0000", 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.338060+0000", 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.338067+0000", 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5320000000000001e-06 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.069 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.338086+0000", 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8992999999999999e-05 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.338094+0000", 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1149999999999994e-06 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.349057+0000", 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010962871000000001 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.349460+0000", 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00040309200000000002 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.353546+0000", 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0040859440000000002 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.366265+0000", 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012718444000000001 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.373608+0000", 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0073435139999999998 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.558500+0000", 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.18489136 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.558559+0000", 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9642000000000003e-05 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.559787+0000", 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001228214 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:192445 8.4 8:2715e38a:::benchmark_data_smithi032_233595_object92600:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:50.347236+0000", 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 7.6684201390000002, 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.21371022000000001, 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.073 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 192445 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.347236+0000", 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.347236+0000", 2024-09-21T10:23:58.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.347235+0000", 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.347242+0000", 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7680000000000003e-06 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.347254+0000", 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2522e-05 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.347262+0000", 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8150000000000006e-06 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.357761+0000", 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010498416999999999 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.357891+0000", 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013091099999999999 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.358026+0000", 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000134497 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.370545+0000", 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012519069000000001 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.379025+0000", 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0084799939999999994 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.559895+0000", 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.18087007599999999 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.559952+0000", 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6786999999999997e-05 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.560947+0000", 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00099471200000000003 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:192455 8.4 8:29727912:::benchmark_data_smithi032_233595_object92610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:50.356815+0000", 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 7.6588410710000003, 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.20514465900000001, 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.079 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 192455 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.356815+0000", 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.356815+0000", 2024-09-21T10:23:58.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.356814+0000", 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.356821+0000", 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5740000000000001e-06 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.356837+0000", 2024-09-21T10:23:58.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5897e-05 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.356844+0000", 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7470000000000005e-06 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.362490+0000", 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0056453620000000001 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.362628+0000", 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013851799999999999 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.362782+0000", 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000153764 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.371907+0000", 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0091254620000000009 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.390143+0000", 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018236013999999998 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.561003+0000", 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.17085925699999999 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.561063+0000", 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0288000000000003e-05 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.561960+0000", 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00089709800000000004 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:192458 8.4 8:298cfcff:::benchmark_data_smithi032_233595_object92613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:50.362166+0000", 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 7.6534903940000003, 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.209991386, 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.085 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 192458 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.362166+0000", 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.362166+0000", 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.362164+0000", 2024-09-21T10:23:58.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:23:58.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.362171+0000", 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0269999999999998e-06 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.362189+0000", 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7850999999999999e-05 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.362197+0000", 2024-09-21T10:23:58.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5599999999999996e-06 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.366345+0000", 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0041484729999999997 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.366497+0000", 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0001518 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.366642+0000", 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000145365 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.376399+0000", 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0097561590000000004 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.397551+0000", 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.021151992000000001 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.570550+0000", 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.17299948400000001 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.570624+0000", 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3743000000000002e-05 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.572157+0000", 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015337230000000001 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.091 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:192475 8.4 8:205bba3a:::benchmark_data_smithi032_233595_object92630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:50.384275+0000", 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 7.6313810980000003, 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.188942675, 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 192475 2024-09-21T10:23:58.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.384275+0000", 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.384275+0000", 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.384273+0000", 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.384282+0000", 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7900000000000005e-06 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.384300+0000", 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7750000000000001e-05 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.384308+0000", 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1149999999999994e-06 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.384393+0000", 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5384999999999996e-05 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.384789+0000", 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000395967 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.385281+0000", 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00049156000000000004 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.390367+0000", 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0050858589999999999 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.402847+0000", 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012479885 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.096 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.572219+0000", 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.169371932 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.572274+0000", 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5427999999999999e-05 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:50.573218+0000", 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000944106 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.097 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:194078 8.4 8:23505c10:::benchmark_data_smithi032_233595_object94233:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:52.366062+0000", 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 5.6495948279999997, 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.076212878999999997, 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 194078 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.366062+0000", 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.366062+0000", 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.366060+0000", 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.366066+0000", 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0390000000000001e-06 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.366083+0000", 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6239000000000002e-05 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.366090+0000", 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3560000000000004e-06 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.400299+0000", 2024-09-21T10:23:58.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.034208781000000001 2024-09-21T10:23:58.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.101 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:57 smithi045 ceph-mon[95976]: pgmap v1109: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 255 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 11235/12120 objects degraded (92.698%); 279/12120 objects misplaced (2.302%); 5.9 MiB/s, 182 objects/s recovering 2024-09-21T10:23:58.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.400449+0000", 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015013 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.400596+0000", 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000147138 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.414590+0000", 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013993759999999999 2024-09-21T10:23:58.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.424734+0000", 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010143838000000001 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.440679+0000", 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015945325 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.440957+0000", 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00027789299999999998 2024-09-21T10:23:58.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.442275+0000", 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013175999999999999 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:194080 8.4 8:26240ae3:::benchmark_data_smithi032_233595_object94235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:52.369644+0000", 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 5.6460128799999998, 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.078384933000000004, 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.104 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 194080 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.369644+0000", 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.369644+0000", 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.369642+0000", 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.369649+0000", 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2250000000000003e-06 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.369664+0000", 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5182e-05 2024-09-21T10:23:58.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.369673+0000", 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7110000000000001e-06 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.418459+0000", 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.048785691999999999 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.418605+0000", 2024-09-21T10:23:58.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014619000000000001 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.418752+0000", 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014687500000000001 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.428973+0000", 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010220808 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.432912+0000", 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039395080000000004 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.446667+0000", 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013754990999999999 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.446734+0000", 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7413e-05 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.109 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.448029+0000", 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001294087 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:194212 8.4 8:2ff011c7:::benchmark_data_smithi032_233595_object94367:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:52.527241+0000", 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 5.4884154499999998, 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.069090289999999999, 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.110 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 194212 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.527241+0000", 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.527241+0000", 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.527240+0000", 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.527246+0000", 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.105e-06 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.527265+0000", 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9378999999999999e-05 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.527272+0000", 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2559999999999996e-06 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.527346+0000", 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4000999999999998e-05 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.527501+0000", 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000154449 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.527639+0000", 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013809900000000001 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.533047+0000", 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0054077350000000003 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.549907+0000", 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.016860796000000001 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.594678+0000", 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.044770096000000002 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.594781+0000", 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000103527 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.596331+0000", 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001550262 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.115 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:194213 8.4 8:23b5729e:::benchmark_data_smithi032_233595_object94368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:52.532915+0000", 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 5.4827416180000004, 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.067799794999999996, 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 194213 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.116 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.532915+0000", 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.532915+0000", 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.532913+0000", 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999976 2024-09-21T10:23:58.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.532921+0000", 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7369999999999995e-06 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.532935+0000", 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3338e-05 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.532942+0000", 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7139999999999993e-06 2024-09-21T10:23:58.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.533070+0000", 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012798900000000001 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.533209+0000", 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013813399999999999 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.533363+0000", 2024-09-21T10:23:58.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015478399999999999 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.538507+0000", 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0051437160000000004 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.549930+0000", 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011422856 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.599318+0000", 2024-09-21T10:23:58.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.049388207000000003 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.599379+0000", 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1420999999999994e-05 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:52.600715+0000", 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013351890000000001 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.121 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:194683 8.2 8:48e86518:::benchmark_data_smithi032_233595_object94838:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:53.180406+0000", 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.8352510019999997, 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.098878474999999993, 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 194683 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.122 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.180406+0000", 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.180406+0000", 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.180404+0000", 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.180411+0000", 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5449999999999998e-06 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.180421+0000", 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0366e-05 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.180429+0000", 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2230000000000001e-06 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.182260+0000", 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001831016 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.182425+0000", 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016552299999999999 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.182581+0000", 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015627800000000001 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.192820+0000", 2024-09-21T10:23:58.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010238498 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.202059+0000", 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0092395099999999994 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.279200+0000", 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.077140021000000003 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.279253+0000", 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3838000000000002e-05 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.279284+0000", 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0617999999999997e-05 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.128 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:194686 8.2 8:487c5932:::benchmark_data_smithi032_233595_object94841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.128 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:53.183751+0000", 2024-09-21T10:23:58.128 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.8319055970000004, 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.095646137000000006, 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 194686 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.183751+0000", 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.129 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.183751+0000", 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.183749+0000", 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.183757+0000", 2024-09-21T10:23:58.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4030000000000007e-06 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.183776+0000", 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8179e-05 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.183784+0000", 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7199999999999995e-06 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.187021+0000", 2024-09-21T10:23:58.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0032361930000000001 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.187433+0000", 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00041205799999999998 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.187834+0000", 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00040103699999999998 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.192852+0000", 2024-09-21T10:23:58.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0050179650000000001 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.202746+0000", 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0098940420000000005 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.279318+0000", 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.076571949 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.279348+0000", 2024-09-21T10:23:58.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.057e-05 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.279397+0000", 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8857000000000003e-05 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:194789 8.4 8:2213a85b:::benchmark_data_smithi032_233595_object94944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:53.379492+0000", 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4.6361642329999997, 2024-09-21T10:23:58.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.068570238000000006, 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 194789 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.379492+0000", 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.379492+0000", 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.379490+0000", 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.379498+0000", 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6499999999999996e-06 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.379520+0000", 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2039e-05 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.379528+0000", 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0879999999999995e-06 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.381344+0000", 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0018155980000000001 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.381696+0000", 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00035206799999999998 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.382235+0000", 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00053938300000000005 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.391813+0000", 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0095772970000000002 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.399924+0000", 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0081114050000000004 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.446676+0000", 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.046751982999999997 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.446781+0000", 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000104806 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:53.448063+0000", 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0012817810000000001 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:195404 8.4 8:2874ff32:::benchmark_data_smithi032_233595_object95559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:54.157457+0000", 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 3.8581994150000001, 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.076077471999999993, 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:23:58.140 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 195404 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.157457+0000", 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.157457+0000", 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.157456+0000", 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.157462+0000", 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6000000000000003e-06 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.157473+0000", 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0693000000000001e-05 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.157480+0000", 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9009999999999997e-06 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.182220+0000", 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.024740539999999998 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.182367+0000", 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014653499999999999 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.182522+0000", 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000155033 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.191003+0000", 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0084808820000000004 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.201391+0000", 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010388158 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.232210+0000", 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.030818805000000001 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.232273+0000", 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3659999999999997e-05 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:54.233535+0000", 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0012610849999999999 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.145 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.146 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:23:58.146 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:23:58.146 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:23:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.146 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:58.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:57 smithi032 ceph-mon[101299]: pgmap v1109: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 255 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 11235/12120 objects degraded (92.698%); 279/12120 objects misplaced (2.302%); 5.9 MiB/s, 182 objects/s recovering 2024-09-21T10:23:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:57 smithi032 ceph-mon[105748]: pgmap v1109: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 255 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 11235/12120 objects degraded (92.698%); 279/12120 objects misplaced (2.302%); 5.9 MiB/s, 182 objects/s recovering 2024-09-21T10:23:58.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:58.289 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:58.391 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:58.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.434 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:58.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:58.534 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:58.577 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:58.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:58.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:58.725 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:58.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:58.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:58.850 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:58.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:58.952 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:58.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:59.081 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:59.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.115 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:59.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:59.216 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:59.223 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:23:59.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.254 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:59.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:59.355 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:59.396 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:23:59.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:59.399 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Removed 99842 objects 2024-09-21T10:23:59.399 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Clean up completed and total clean up time :173.363 2024-09-21T10:23:59.497 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:59.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.521 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:59.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:59.551 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:23:59.622 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:59.644 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:59.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:59.700 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:23:59.701 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:23:59.746 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:59.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:59.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:59.826 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:23:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:23:59] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-21T10:23:59.833 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:23:59.871 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:23:59.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:23:59.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:23:59.951 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:59 smithi045 ceph-mon[95976]: pgmap v1110: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 221 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 11235/10497 objects degraded (107.031%); 279/10497 objects misplaced (2.658%); 4.8 MiB/s, 149 objects/s recovering 2024-09-21T10:23:59.951 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1077850124' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:23:59.951 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:23:59 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:23:59.996 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:00.019 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:00.121 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:00.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.144 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:00.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:59 smithi032 ceph-mon[101299]: pgmap v1110: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 221 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 11235/10497 objects degraded (107.031%); 279/10497 objects misplaced (2.658%); 4.8 MiB/s, 149 objects/s recovering 2024-09-21T10:24:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1077850124' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:24:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:23:59 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:24:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:59 smithi032 ceph-mon[105748]: pgmap v1110: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 221 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 11235/10497 objects degraded (107.031%); 279/10497 objects misplaced (2.658%); 4.8 MiB/s, 149 objects/s recovering 2024-09-21T10:24:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1077850124' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:24:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:23:59 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:24:00.245 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:00.267 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:00.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:00.368 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:00.391 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:00.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:00.493 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:00.515 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:00.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:00.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:00.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:00.766 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:00.868 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:00.891 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:00.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:01.015 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:01.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:00 smithi045 ceph-mon[95976]: from='client.35862 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:01.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:00 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:01.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:24:01.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:00 smithi045 ceph-mon[95976]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T10:24:01.117 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:01.139 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:01.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:00 smithi032 ceph-mon[105748]: from='client.35862 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:00 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:24:01.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:00 smithi032 ceph-mon[105748]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T10:24:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:00 smithi032 ceph-mon[101299]: from='client.35862 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:00 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:24:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:00 smithi032 ceph-mon[101299]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T10:24:01.241 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:01.264 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:01.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:01.326 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:24:01.327 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:24:01.327 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 0 2024-09-21T10:24:01.327 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:24:01.365 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:01.387 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1110,"stamp":"2024-09-21T10:23:58.551045+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":232200189,"num_objects":3499,"num_object_clones":0,"num_object_copies":10497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11235,"num_objects_misplaced":279,"num_objects_unfound":0,"num_objects_dirty":3499,"num_whiteouts":0,"num_read":6389,"num_read_kb":13844,"num_write":197338,"num_write_kb":6404930,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28030,"num_bytes_recovered":1453622391,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113659,"ondisk_log_size":113659,"up":297,"acting":292,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6235588,"kb_used_data":1543556,"kb_used_omap":40,"kb_used_meta":4691735,"kb_avail":743725628,"statfs":{"total":767960285184,"available":761575043072,"internally_reserved":0,"allocated":1580601344,"data_stored":1565350693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41824,"internal_metadata":4804336800},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,4,3,11,10],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-461111296,"num_objects":-7036,"num_object_clones":0,"num_object_copies":-21108,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":-994,"num_objects_misplaced":-732,"num_objects_unfound":0,"num_objects_dirty":-7036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7036,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1462,"num_bytes_recovered":49414712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.784250"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":611,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.773595+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:23:49.773595+0000","last_peered":"2024-09-21T10:23:49.773595+0000","last_clean":"2024-09-21T10:23:49.773595+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:23:49.773595+0000","last_undegraded":"2024-09-21T10:23:49.773595+0000","last_fullsized":"2024-09-21T10:23:49.773595+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":370,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.771205+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:23:48.771205+0000","last_peered":"2024-09-21T10:23:48.771205+0000","last_clean":"2024-09-21T10:23:48.771205+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:23:48.771205+0000","last_undegraded":"2024-09-21T10:23:48.771205+0000","last_fullsized":"2024-09-21T10:23:48.771205+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768714+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:23:48.768714+0000","last_peered":"2024-09-21T10:23:48.768714+0000","last_clean":"2024-09-21T10:23:48.768714+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:23:48.768714+0000","last_undegraded":"2024-09-21T10:23:48.768714+0000","last_fullsized":"2024-09-21T10:23:48.768714+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.775713+0000","last_change":"2024-09-21T10:22:01.318638+0000","last_active":"2024-09-21T10:23:49.775713+0000","last_peered":"2024-09-21T10:23:49.775713+0000","last_clean":"2024-09-21T10:23:49.775713+0000","last_became_active":"2024-09-21T10:22:01.317087+0000","last_became_peered":"2024-09-21T10:22:01.317087+0000","last_unstale":"2024-09-21T10:23:49.775713+0000","last_undegraded":"2024-09-21T10:23:49.775713+0000","last_fullsized":"2024-09-21T10:23:49.775713+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:25.334726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.776247+0000","last_change":"2024-09-21T10:22:01.318978+0000","last_active":"2024-09-21T10:23:49.776247+0000","last_peered":"2024-09-21T10:23:49.776247+0000","last_clean":"2024-09-21T10:23:49.776247+0000","last_became_active":"2024-09-21T10:22:01.318255+0000","last_became_peered":"2024-09-21T10:22:01.318255+0000","last_unstale":"2024-09-21T10:23:49.776247+0000","last_undegraded":"2024-09-21T10:23:49.776247+0000","last_fullsized":"2024-09-21T10:23:49.776247+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:55.084088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":341,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770479+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:23:48.770479+0000","last_peered":"2024-09-21T10:23:48.770479+0000","last_clean":"2024-09-21T10:23:48.770479+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:23:48.770479+0000","last_undegraded":"2024-09-21T10:23:48.770479+0000","last_fullsized":"2024-09-21T10:23:48.770479+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":632,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.773550+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:23:49.773550+0000","last_peered":"2024-09-21T10:23:49.773550+0000","last_clean":"2024-09-21T10:23:49.773550+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:23:49.773550+0000","last_undegraded":"2024-09-21T10:23:49.773550+0000","last_fullsized":"2024-09-21T10:23:49.773550+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":422,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768655+0000","last_change":"2024-09-21T10:22:01.331612+0000","last_active":"2024-09-21T10:23:48.768655+0000","last_peered":"2024-09-21T10:23:48.768655+0000","last_clean":"2024-09-21T10:23:48.768655+0000","last_became_active":"2024-09-21T10:22:01.325282+0000","last_became_peered":"2024-09-21T10:22:01.325282+0000","last_unstale":"2024-09-21T10:23:48.768655+0000","last_undegraded":"2024-09-21T10:23:48.768655+0000","last_fullsized":"2024-09-21T10:23:48.768655+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:23.383106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"367'12295","reported_seq":13368,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:55.087091+0000","last_change":"2024-09-21T10:21:39.510930+0000","last_active":"2024-09-21T10:23:55.087091+0000","last_peered":"2024-09-21T10:23:55.087091+0000","last_clean":"2024-09-21T10:20:12.056605+0000","last_became_active":"2024-09-21T10:20:14.466147+0000","last_became_peered":"2024-09-21T10:20:14.466147+0000","last_unstale":"2024-09-21T10:23:55.087091+0000","last_undegraded":"2024-09-21T10:20:14.094800+0000","last_fullsized":"2024-09-21T10:20:14.071992+0000","mapping_epoch":325,"log_start":"329'4900","ondisk_log_start":"329'4900","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7395,"log_dups_size":0,"ondisk_log_size":7395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:34.237367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14745600,"num_objects":225,"num_object_clones":0,"num_object_copies":675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3167,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12295,"num_write_kb":400640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":489,"num_bytes_recovered":32047104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-2942}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":57,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.819114+0000","last_change":"2024-09-21T10:22:01.321467+0000","last_active":"2024-09-21T10:23:49.819114+0000","last_peered":"2024-09-21T10:23:49.819114+0000","last_clean":"2024-09-21T10:23:49.819114+0000","last_became_active":"2024-09-21T10:22:01.318270+0000","last_became_peered":"2024-09-21T10:22:01.318270+0000","last_unstale":"2024-09-21T10:23:49.819114+0000","last_undegraded":"2024-09-21T10:23:49.819114+0000","last_fullsized":"2024-09-21T10:23:49.819114+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:44.059553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":74,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.767893+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:23:48.767893+0000","last_peered":"2024-09-21T10:23:48.767893+0000","last_clean":"2024-09-21T10:23:48.767893+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:23:48.767893+0000","last_undegraded":"2024-09-21T10:23:48.767893+0000","last_fullsized":"2024-09-21T10:23:48.767893+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"367'12038","reported_seq":15921,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:54.996911+0000","last_change":"2024-09-21T10:20:04.684629+0000","last_active":"2024-09-21T10:23:54.996911+0000","last_peered":"2024-09-21T10:23:54.996911+0000","last_clean":"2024-09-21T10:23:54.996911+0000","last_became_active":"2024-09-21T10:20:04.680299+0000","last_became_peered":"2024-09-21T10:20:04.680299+0000","last_unstale":"2024-09-21T10:23:54.996911+0000","last_undegraded":"2024-09-21T10:23:54.996911+0000","last_fullsized":"2024-09-21T10:23:54.996911+0000","mapping_epoch":320,"log_start":"333'5300","ondisk_log_start":"333'5300","created":305,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6738,"log_dups_size":0,"ondisk_log_size":6738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:41.743081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13893632,"num_objects":212,"num_object_clones":0,"num_object_copies":636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12038,"num_write_kb":392000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1891,"num_bytes_recovered":123928576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":44,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.819386+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:23:49.819386+0000","last_peered":"2024-09-21T10:23:49.819386+0000","last_clean":"2024-09-21T10:23:49.819386+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:23:49.819386+0000","last_undegraded":"2024-09-21T10:23:49.819386+0000","last_fullsized":"2024-09-21T10:23:49.819386+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":63,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819051+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:23:48.819051+0000","last_peered":"2024-09-21T10:23:48.819051+0000","last_clean":"2024-09-21T10:23:48.819051+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:23:48.819051+0000","last_undegraded":"2024-09-21T10:23:48.819051+0000","last_fullsized":"2024-09-21T10:23:48.819051+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"367'12167","reported_seq":12304,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:56.683787+0000","last_change":"2024-09-21T10:19:22.649712+0000","last_active":"2024-09-21T10:23:56.683787+0000","last_peered":"2024-09-21T10:23:56.683787+0000","last_clean":"2024-09-21T10:23:56.683787+0000","last_became_active":"2024-09-21T10:19:22.649303+0000","last_became_peered":"2024-09-21T10:19:22.649303+0000","last_unstale":"2024-09-21T10:23:56.683787+0000","last_undegraded":"2024-09-21T10:23:56.683787+0000","last_fullsized":"2024-09-21T10:23:56.683787+0000","mapping_epoch":305,"log_start":"331'4961","ondisk_log_start":"331'4961","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7206,"log_dups_size":0,"ondisk_log_size":7206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:24.442851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8978432,"num_objects":137,"num_object_clones":0,"num_object_copies":411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12167,"num_write_kb":393728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":365,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770891+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:23:48.770891+0000","last_peered":"2024-09-21T10:23:48.770891+0000","last_clean":"2024-09-21T10:23:48.770891+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:23:48.770891+0000","last_undegraded":"2024-09-21T10:23:48.770891+0000","last_fullsized":"2024-09-21T10:23:48.770891+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":385,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768152+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:23:48.768152+0000","last_peered":"2024-09-21T10:23:48.768152+0000","last_clean":"2024-09-21T10:23:48.768152+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:23:48.768152+0000","last_undegraded":"2024-09-21T10:23:48.768152+0000","last_fullsized":"2024-09-21T10:23:48.768152+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"367'12641","reported_seq":12778,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:55.447522+0000","last_change":"2024-09-21T10:19:22.640645+0000","last_active":"2024-09-21T10:23:55.447522+0000","last_peered":"2024-09-21T10:23:55.447522+0000","last_clean":"2024-09-21T10:23:55.447522+0000","last_became_active":"2024-09-21T10:19:22.640343+0000","last_became_peered":"2024-09-21T10:19:22.640343+0000","last_unstale":"2024-09-21T10:23:55.447522+0000","last_undegraded":"2024-09-21T10:23:55.447522+0000","last_fullsized":"2024-09-21T10:23:55.447522+0000","mapping_epoch":305,"log_start":"337'7000","ondisk_log_start":"337'7000","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5641,"log_dups_size":0,"ondisk_log_size":5641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:48.734122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13172736,"num_objects":201,"num_object_clones":0,"num_object_copies":603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12641,"num_write_kb":410944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":299,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768387+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:23:48.768387+0000","last_peered":"2024-09-21T10:23:48.768387+0000","last_clean":"2024-09-21T10:23:48.768387+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:23:48.768387+0000","last_undegraded":"2024-09-21T10:23:48.768387+0000","last_fullsized":"2024-09-21T10:23:48.768387+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":223,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.775886+0000","last_change":"2024-09-21T10:19:50.585724+0000","last_active":"2024-09-21T10:23:49.775886+0000","last_peered":"2024-09-21T10:23:49.775886+0000","last_clean":"2024-09-21T10:23:49.775886+0000","last_became_active":"2024-09-21T10:19:50.585315+0000","last_became_peered":"2024-09-21T10:19:50.585315+0000","last_unstale":"2024-09-21T10:23:49.775886+0000","last_undegraded":"2024-09-21T10:23:49.775886+0000","last_fullsized":"2024-09-21T10:23:49.775886+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:26.618970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"367'12251","reported_seq":17351,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:53.903034+0000","last_change":"2024-09-21T10:22:01.328242+0000","last_active":"2024-09-21T10:23:53.903034+0000","last_peered":"2024-09-21T10:23:53.903034+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492359+0000","last_became_peered":"2024-09-21T10:20:14.492359+0000","last_unstale":"2024-09-21T10:23:53.903034+0000","last_undegraded":"2024-09-21T10:20:14.218670+0000","last_fullsized":"2024-09-21T10:20:14.071510+0000","mapping_epoch":325,"log_start":"333'5400","ondisk_log_start":"333'5400","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6851,"log_dups_size":0,"ondisk_log_size":6851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17891328,"num_objects":273,"num_object_clones":0,"num_object_copies":819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12251,"num_write_kb":400768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2481,"num_bytes_recovered":148307968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-931}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":622,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.818084+0000","last_change":"2024-09-21T10:22:04.029688+0000","last_active":"2024-09-21T10:23:49.818084+0000","last_peered":"2024-09-21T10:23:49.818084+0000","last_clean":"2024-09-21T10:23:49.818084+0000","last_became_active":"2024-09-21T10:22:04.028853+0000","last_became_peered":"2024-09-21T10:22:04.028853+0000","last_unstale":"2024-09-21T10:23:49.818084+0000","last_undegraded":"2024-09-21T10:23:49.818084+0000","last_fullsized":"2024-09-21T10:23:49.818084+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:20.457596+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:19:20.457596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":661,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.773171+0000","last_change":"2024-09-21T10:22:01.335303+0000","last_active":"2024-09-21T10:23:49.773171+0000","last_peered":"2024-09-21T10:23:49.773171+0000","last_clean":"2024-09-21T10:23:49.773171+0000","last_became_active":"2024-09-21T10:22:01.325298+0000","last_became_peered":"2024-09-21T10:22:01.325298+0000","last_unstale":"2024-09-21T10:23:49.773171+0000","last_undegraded":"2024-09-21T10:23:49.773171+0000","last_fullsized":"2024-09-21T10:23:49.773171+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:01.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":63,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819212+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:23:48.819212+0000","last_peered":"2024-09-21T10:23:48.819212+0000","last_clean":"2024-09-21T10:23:48.819212+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:23:48.819212+0000","last_undegraded":"2024-09-21T10:23:48.819212+0000","last_fullsized":"2024-09-21T10:23:48.819212+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"367'12009","reported_seq":12146,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:55.466287+0000","last_change":"2024-09-21T10:19:22.641281+0000","last_active":"2024-09-21T10:23:55.466287+0000","last_peered":"2024-09-21T10:23:55.466287+0000","last_clean":"2024-09-21T10:23:55.466287+0000","last_became_active":"2024-09-21T10:19:22.640988+0000","last_became_peered":"2024-09-21T10:19:22.640988+0000","last_unstale":"2024-09-21T10:23:55.466287+0000","last_undegraded":"2024-09-21T10:23:55.466287+0000","last_fullsized":"2024-09-21T10:23:55.466287+0000","mapping_epoch":305,"log_start":"336'6400","ondisk_log_start":"336'6400","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5609,"log_dups_size":0,"ondisk_log_size":5609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:52.652565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12648448,"num_objects":193,"num_object_clones":0,"num_object_copies":579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12009,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":449,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770294+0000","last_change":"2024-09-21T10:23:47.785204+0000","last_active":"2024-09-21T10:23:48.770294+0000","last_peered":"2024-09-21T10:23:48.770294+0000","last_clean":"2024-09-21T10:23:48.770294+0000","last_became_active":"2024-09-21T10:23:47.784780+0000","last_became_peered":"2024-09-21T10:23:47.784780+0000","last_unstale":"2024-09-21T10:23:48.770294+0000","last_undegraded":"2024-09-21T10:23:48.770294+0000","last_fullsized":"2024-09-21T10:23:48.770294+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_clean_scrub_stamp":"2024-09-21T10:19:15.365774+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":375,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768320+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:23:48.768320+0000","last_peered":"2024-09-21T10:23:48.768320+0000","last_clean":"2024-09-21T10:23:48.768320+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:23:48.768320+0000","last_undegraded":"2024-09-21T10:23:48.768320+0000","last_fullsized":"2024-09-21T10:23:48.768320+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":63,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819702+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:23:48.819702+0000","last_peered":"2024-09-21T10:23:48.819702+0000","last_clean":"2024-09-21T10:23:48.819702+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:23:48.819702+0000","last_undegraded":"2024-09-21T10:23:48.819702+0000","last_fullsized":"2024-09-21T10:23:48.819702+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"367'12378","reported_seq":19717,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:55.072875+0000","last_change":"2024-09-21T10:22:34.604937+0000","last_active":"2024-09-21T10:23:55.072875+0000","last_peered":"2024-09-21T10:23:55.072875+0000","last_clean":"2024-09-21T10:23:55.072875+0000","last_became_active":"2024-09-21T10:22:34.577215+0000","last_became_peered":"2024-09-21T10:22:34.577215+0000","last_unstale":"2024-09-21T10:23:55.072875+0000","last_undegraded":"2024-09-21T10:23:55.072875+0000","last_fullsized":"2024-09-21T10:23:55.072875+0000","mapping_epoch":354,"log_start":"329'5000","ondisk_log_start":"329'5000","created":305,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7378,"log_dups_size":0,"ondisk_log_size":7378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:40.287923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15466496,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12378,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3618,"num_bytes_recovered":121634816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":63,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.818962+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:23:48.818962+0000","last_peered":"2024-09-21T10:23:48.818962+0000","last_clean":"2024-09-21T10:23:48.818962+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:23:48.818962+0000","last_undegraded":"2024-09-21T10:23:48.818962+0000","last_fullsized":"2024-09-21T10:23:48.818962+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":63,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.820532+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:23:48.820532+0000","last_peered":"2024-09-21T10:23:48.820532+0000","last_clean":"2024-09-21T10:23:48.820532+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:23:48.820532+0000","last_undegraded":"2024-09-21T10:23:48.820532+0000","last_fullsized":"2024-09-21T10:23:48.820532+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"367'12341","reported_seq":13603,"reported_epoch":367,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:55.475077+0000","last_change":"2024-09-21T10:23:46.550830+0000","last_active":"2024-09-21T10:23:55.475077+0000","last_peered":"2024-09-21T10:23:55.475077+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.492538+0000","last_became_peered":"2024-09-21T10:20:14.492538+0000","last_unstale":"2024-09-21T10:23:55.475077+0000","last_undegraded":"2024-09-21T10:20:14.216051+0000","last_fullsized":"2024-09-21T10:20:14.075057+0000","mapping_epoch":325,"log_start":"336'6700","ondisk_log_start":"336'6700","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5641,"log_dups_size":0,"ondisk_log_size":5641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11862016,"num_objects":181,"num_object_clones":0,"num_object_copies":543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3022,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12341,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":559,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-2841}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":622,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.818057+0000","last_change":"2024-09-21T10:22:04.031320+0000","last_active":"2024-09-21T10:23:49.818057+0000","last_peered":"2024-09-21T10:23:49.818057+0000","last_clean":"2024-09-21T10:23:49.818057+0000","last_became_active":"2024-09-21T10:22:04.030774+0000","last_became_peered":"2024-09-21T10:22:04.030774+0000","last_unstale":"2024-09-21T10:23:49.818057+0000","last_undegraded":"2024-09-21T10:23:49.818057+0000","last_fullsized":"2024-09-21T10:23:49.818057+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:22.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:22.489745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":640,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.772803+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:23:49.772803+0000","last_peered":"2024-09-21T10:23:49.772803+0000","last_clean":"2024-09-21T10:23:49.772803+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:23:49.772803+0000","last_undegraded":"2024-09-21T10:23:49.772803+0000","last_fullsized":"2024-09-21T10:23:49.772803+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":332,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.781587+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:23:49.781587+0000","last_peered":"2024-09-21T10:23:49.781587+0000","last_clean":"2024-09-21T10:23:49.781587+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:23:49.781587+0000","last_undegraded":"2024-09-21T10:23:49.781587+0000","last_fullsized":"2024-09-21T10:23:49.781587+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"367'12243","reported_seq":20691,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:54.992440+0000","last_change":"2024-09-21T10:23:49.804644+0000","last_active":"2024-09-21T10:23:54.992440+0000","last_peered":"2024-09-21T10:23:54.992440+0000","last_clean":"2024-09-21T10:23:54.992440+0000","last_became_active":"2024-09-21T10:23:49.776537+0000","last_became_peered":"2024-09-21T10:23:49.776537+0000","last_unstale":"2024-09-21T10:23:54.992440+0000","last_undegraded":"2024-09-21T10:23:54.992440+0000","last_fullsized":"2024-09-21T10:23:54.992440+0000","mapping_epoch":366,"log_start":"334'5500","ondisk_log_start":"334'5500","created":305,"last_epoch_clean":367,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:07:11.327754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13041664,"num_objects":199,"num_object_clones":0,"num_object_copies":597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12243,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4170,"num_bytes_recovered":155779072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":622,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770306+0000","last_change":"2024-09-21T10:22:04.029284+0000","last_active":"2024-09-21T10:23:48.770306+0000","last_peered":"2024-09-21T10:23:48.770306+0000","last_clean":"2024-09-21T10:23:48.770306+0000","last_became_active":"2024-09-21T10:22:04.028938+0000","last_became_peered":"2024-09-21T10:22:04.028938+0000","last_unstale":"2024-09-21T10:23:48.770306+0000","last_undegraded":"2024-09-21T10:23:48.770306+0000","last_fullsized":"2024-09-21T10:23:48.770306+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:12.388304+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:12.388304+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":651,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:48.768097+0000","last_change":"2024-09-21T10:21:36.010162+0000","last_active":"2024-09-21T10:23:48.768097+0000","last_peered":"2024-09-21T10:23:48.768097+0000","last_clean":"2024-09-21T10:21:17.725138+0000","last_became_active":"2024-09-21T10:21:36.009706+0000","last_became_peered":"2024-09-21T10:21:36.009706+0000","last_unstale":"2024-09-21T10:23:48.768097+0000","last_undegraded":"2024-09-21T10:21:35.852171+0000","last_fullsized":"2024-09-21T10:21:35.851526+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:04.206298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":571,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.772849+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:23:49.772849+0000","last_peered":"2024-09-21T10:23:49.772849+0000","last_clean":"2024-09-21T10:23:49.772849+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:23:49.772849+0000","last_undegraded":"2024-09-21T10:23:49.772849+0000","last_fullsized":"2024-09-21T10:23:49.772849+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":342,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.776044+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:23:49.776044+0000","last_peered":"2024-09-21T10:23:49.776044+0000","last_clean":"2024-09-21T10:23:49.776044+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:23:49.776044+0000","last_undegraded":"2024-09-21T10:23:49.776044+0000","last_fullsized":"2024-09-21T10:23:49.776044+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"367'12012","reported_seq":12149,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:23:55.470615+0000","last_change":"2024-09-21T10:20:14.480108+0000","last_active":"2024-09-21T10:23:55.470615+0000","last_peered":"2024-09-21T10:23:55.470615+0000","last_clean":"2024-09-21T10:19:21.625671+0000","last_became_active":"2024-09-21T10:20:14.471864+0000","last_became_peered":"2024-09-21T10:20:14.471864+0000","last_unstale":"2024-09-21T10:23:55.470615+0000","last_undegraded":"2024-09-21T10:20:14.214064+0000","last_fullsized":"2024-09-21T10:20:14.074889+0000","mapping_epoch":325,"log_start":"336'6400","ondisk_log_start":"336'6400","created":305,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":5612,"log_dups_size":0,"ondisk_log_size":5612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12582912,"num_objects":192,"num_object_clones":0,"num_object_copies":576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12012,"num_write_kb":390528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-3353}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":622,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770937+0000","last_change":"2024-09-21T10:22:04.031573+0000","last_active":"2024-09-21T10:23:48.770937+0000","last_peered":"2024-09-21T10:23:48.770937+0000","last_clean":"2024-09-21T10:23:48.770937+0000","last_became_active":"2024-09-21T10:22:04.031125+0000","last_became_peered":"2024-09-21T10:22:04.031125+0000","last_unstale":"2024-09-21T10:23:48.770937+0000","last_undegraded":"2024-09-21T10:23:48.770937+0000","last_fullsized":"2024-09-21T10:23:48.770937+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:19:19.450903+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1007,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.771147+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:23:48.771147+0000","last_peered":"2024-09-21T10:23:48.771147+0000","last_clean":"2024-09-21T10:23:48.771147+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:23:48.771147+0000","last_undegraded":"2024-09-21T10:23:48.771147+0000","last_fullsized":"2024-09-21T10:23:48.771147+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":221,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.775914+0000","last_change":"2024-09-21T10:19:51.555124+0000","last_active":"2024-09-21T10:23:49.775914+0000","last_peered":"2024-09-21T10:23:49.775914+0000","last_clean":"2024-09-21T10:23:49.775914+0000","last_became_active":"2024-09-21T10:19:51.554766+0000","last_became_peered":"2024-09-21T10:19:51.554766+0000","last_unstale":"2024-09-21T10:23:49.775914+0000","last_undegraded":"2024-09-21T10:23:49.775914+0000","last_fullsized":"2024-09-21T10:23:49.775914+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:52.832950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":52,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.775799+0000","last_change":"2024-09-21T10:22:01.316140+0000","last_active":"2024-09-21T10:23:49.775799+0000","last_peered":"2024-09-21T10:23:49.775799+0000","last_clean":"2024-09-21T10:23:49.775799+0000","last_became_active":"2024-09-21T10:22:01.315408+0000","last_became_peered":"2024-09-21T10:22:01.315408+0000","last_unstale":"2024-09-21T10:23:49.775799+0000","last_undegraded":"2024-09-21T10:23:49.775799+0000","last_fullsized":"2024-09-21T10:23:49.775799+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:07.322279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"367'12314","reported_seq":16311,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:53.898155+0000","last_change":"2024-09-21T10:20:47.589372+0000","last_active":"2024-09-21T10:23:53.898155+0000","last_peered":"2024-09-21T10:23:53.898155+0000","last_clean":"2024-09-21T10:23:53.898155+0000","last_became_active":"2024-09-21T10:20:47.580051+0000","last_became_peered":"2024-09-21T10:20:47.580051+0000","last_unstale":"2024-09-21T10:23:53.898155+0000","last_undegraded":"2024-09-21T10:23:53.898155+0000","last_fullsized":"2024-09-21T10:23:53.898155+0000","mapping_epoch":332,"log_start":"333'5400","ondisk_log_start":"333'5400","created":305,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6914,"log_dups_size":0,"ondisk_log_size":6914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:46.363318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18612224,"num_objects":284,"num_object_clones":0,"num_object_copies":852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12314,"num_write_kb":403136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":622,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770683+0000","last_change":"2024-09-21T10:22:04.030341+0000","last_active":"2024-09-21T10:23:48.770683+0000","last_peered":"2024-09-21T10:23:48.770683+0000","last_clean":"2024-09-21T10:23:48.770683+0000","last_became_active":"2024-09-21T10:22:04.030000+0000","last_became_peered":"2024-09-21T10:22:04.030000+0000","last_unstale":"2024-09-21T10:23:48.770683+0000","last_undegraded":"2024-09-21T10:23:48.770683+0000","last_fullsized":"2024-09-21T10:23:48.770683+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:10.332770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:10.332770+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1160,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.814814+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:23:49.814814+0000","last_peered":"2024-09-21T10:23:49.814814+0000","last_clean":"2024-09-21T10:23:49.814814+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:23:49.814814+0000","last_undegraded":"2024-09-21T10:23:49.814814+0000","last_fullsized":"2024-09-21T10:23:49.814814+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":319,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.780928+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:23:49.780928+0000","last_peered":"2024-09-21T10:23:49.780928+0000","last_clean":"2024-09-21T10:23:49.780928+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:23:49.780928+0000","last_undegraded":"2024-09-21T10:23:49.780928+0000","last_fullsized":"2024-09-21T10:23:49.780928+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":348,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.781616+0000","last_change":"2024-09-21T10:22:01.317933+0000","last_active":"2024-09-21T10:23:49.781616+0000","last_peered":"2024-09-21T10:23:49.781616+0000","last_clean":"2024-09-21T10:23:49.781616+0000","last_became_active":"2024-09-21T10:22:01.317536+0000","last_became_peered":"2024-09-21T10:22:01.317536+0000","last_unstale":"2024-09-21T10:23:49.781616+0000","last_undegraded":"2024-09-21T10:23:49.781616+0000","last_fullsized":"2024-09-21T10:23:49.781616+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:34.312724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"367'12438","reported_seq":19862,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:55.081817+0000","last_change":"2024-09-21T10:23:46.547468+0000","last_active":"2024-09-21T10:23:55.081817+0000","last_peered":"2024-09-21T10:23:55.081817+0000","last_clean":"2024-09-21T10:23:55.081817+0000","last_became_active":"2024-09-21T10:23:27.674984+0000","last_became_peered":"2024-09-21T10:23:27.674984+0000","last_unstale":"2024-09-21T10:23:55.081817+0000","last_undegraded":"2024-09-21T10:23:55.081817+0000","last_fullsized":"2024-09-21T10:23:55.081817+0000","mapping_epoch":361,"log_start":"327'4200","ondisk_log_start":"327'4200","created":305,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":8238,"log_dups_size":0,"ondisk_log_size":8238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:13.754952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15204352,"num_objects":232,"num_object_clones":0,"num_object_copies":696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12438,"num_write_kb":405440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3641,"num_bytes_recovered":232128512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":622,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770395+0000","last_change":"2024-09-21T10:22:04.029892+0000","last_active":"2024-09-21T10:23:48.770395+0000","last_peered":"2024-09-21T10:23:48.770395+0000","last_clean":"2024-09-21T10:23:48.770395+0000","last_became_active":"2024-09-21T10:22:04.029353+0000","last_became_peered":"2024-09-21T10:22:04.029353+0000","last_unstale":"2024-09-21T10:23:48.770395+0000","last_undegraded":"2024-09-21T10:23:48.770395+0000","last_fullsized":"2024-09-21T10:23:48.770395+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:19:13.371825+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1109,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768788+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:23:48.768788+0000","last_peered":"2024-09-21T10:23:48.768788+0000","last_clean":"2024-09-21T10:23:48.768788+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:23:48.768788+0000","last_undegraded":"2024-09-21T10:23:48.768788+0000","last_fullsized":"2024-09-21T10:23:48.768788+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":63,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819595+0000","last_change":"2024-09-21T10:19:50.581529+0000","last_active":"2024-09-21T10:23:48.819595+0000","last_peered":"2024-09-21T10:23:48.819595+0000","last_clean":"2024-09-21T10:23:48.819595+0000","last_became_active":"2024-09-21T10:19:50.581119+0000","last_became_peered":"2024-09-21T10:19:50.581119+0000","last_unstale":"2024-09-21T10:23:48.819595+0000","last_undegraded":"2024-09-21T10:23:48.819595+0000","last_fullsized":"2024-09-21T10:23:48.819595+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:13.334483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":319,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.781021+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:23:49.781021+0000","last_peered":"2024-09-21T10:23:49.781021+0000","last_clean":"2024-09-21T10:23:49.781021+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:23:49.781021+0000","last_undegraded":"2024-09-21T10:23:49.781021+0000","last_fullsized":"2024-09-21T10:23:49.781021+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"367'12575","reported_seq":16700,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:56.701582+0000","last_change":"2024-09-21T10:20:42.601607+0000","last_active":"2024-09-21T10:23:56.701582+0000","last_peered":"2024-09-21T10:23:56.701582+0000","last_clean":"2024-09-21T10:23:56.701582+0000","last_became_active":"2024-09-21T10:20:42.593406+0000","last_became_peered":"2024-09-21T10:20:42.593406+0000","last_unstale":"2024-09-21T10:23:56.701582+0000","last_undegraded":"2024-09-21T10:23:56.701582+0000","last_fullsized":"2024-09-21T10:23:56.701582+0000","mapping_epoch":330,"log_start":"333'5368","ondisk_log_start":"333'5368","created":305,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":7207,"log_dups_size":0,"ondisk_log_size":7207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:09.770489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9633792,"num_objects":147,"num_object_clones":0,"num_object_copies":441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12575,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1985,"num_bytes_recovered":130088960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":622,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.819232+0000","last_change":"2024-09-21T10:22:04.029371+0000","last_active":"2024-09-21T10:23:49.819232+0000","last_peered":"2024-09-21T10:23:49.819232+0000","last_clean":"2024-09-21T10:23:49.819232+0000","last_became_active":"2024-09-21T10:22:04.029014+0000","last_became_peered":"2024-09-21T10:22:04.029014+0000","last_unstale":"2024-09-21T10:23:49.819232+0000","last_undegraded":"2024-09-21T10:23:49.819232+0000","last_fullsized":"2024-09-21T10:23:49.819232+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":314,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768598+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:23:48.768598+0000","last_peered":"2024-09-21T10:23:48.768598+0000","last_clean":"2024-09-21T10:23:48.768598+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:23:48.768598+0000","last_undegraded":"2024-09-21T10:23:48.768598+0000","last_fullsized":"2024-09-21T10:23:48.768598+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":928,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.781151+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:23:49.781151+0000","last_peered":"2024-09-21T10:23:49.781151+0000","last_clean":"2024-09-21T10:23:49.781151+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:23:49.781151+0000","last_undegraded":"2024-09-21T10:23:49.781151+0000","last_fullsized":"2024-09-21T10:23:49.781151+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":355,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.781183+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:23:49.781183+0000","last_peered":"2024-09-21T10:23:49.781183+0000","last_clean":"2024-09-21T10:23:49.781183+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:23:49.781183+0000","last_undegraded":"2024-09-21T10:23:49.781183+0000","last_fullsized":"2024-09-21T10:23:49.781183+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"367'12076","reported_seq":12213,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:53.916584+0000","last_change":"2024-09-21T10:19:22.641043+0000","last_active":"2024-09-21T10:23:53.916584+0000","last_peered":"2024-09-21T10:23:53.916584+0000","last_clean":"2024-09-21T10:23:53.916584+0000","last_became_active":"2024-09-21T10:19:22.640742+0000","last_became_peered":"2024-09-21T10:19:22.640742+0000","last_unstale":"2024-09-21T10:23:53.916584+0000","last_undegraded":"2024-09-21T10:23:53.916584+0000","last_fullsized":"2024-09-21T10:23:53.916584+0000","mapping_epoch":305,"log_start":"331'5200","ondisk_log_start":"331'5200","created":305,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6876,"log_dups_size":0,"ondisk_log_size":6876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:36.901208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17694720,"num_objects":270,"num_object_clones":0,"num_object_copies":810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12076,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":622,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.819461+0000","last_change":"2024-09-21T10:22:04.031872+0000","last_active":"2024-09-21T10:23:49.819461+0000","last_peered":"2024-09-21T10:23:49.819461+0000","last_clean":"2024-09-21T10:23:49.819461+0000","last_became_active":"2024-09-21T10:22:04.031343+0000","last_became_peered":"2024-09-21T10:22:04.031343+0000","last_unstale":"2024-09-21T10:23:49.819461+0000","last_undegraded":"2024-09-21T10:23:49.819461+0000","last_fullsized":"2024-09-21T10:23:49.819461+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:07.332404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:07.332404+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":337,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.781145+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:23:49.781145+0000","last_peered":"2024-09-21T10:23:49.781145+0000","last_clean":"2024-09-21T10:23:49.781145+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:23:49.781145+0000","last_undegraded":"2024-09-21T10:23:49.781145+0000","last_fullsized":"2024-09-21T10:23:49.781145+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":933,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.771003+0000","last_change":"2024-09-21T10:22:01.336961+0000","last_active":"2024-09-21T10:23:48.771003+0000","last_peered":"2024-09-21T10:23:48.771003+0000","last_clean":"2024-09-21T10:23:48.771003+0000","last_became_active":"2024-09-21T10:22:01.320079+0000","last_became_peered":"2024-09-21T10:22:01.320079+0000","last_unstale":"2024-09-21T10:23:48.771003+0000","last_undegraded":"2024-09-21T10:23:48.771003+0000","last_fullsized":"2024-09-21T10:23:48.771003+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:54.037388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":220,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.775766+0000","last_change":"2024-09-21T10:20:13.088557+0000","last_active":"2024-09-21T10:23:49.775766+0000","last_peered":"2024-09-21T10:23:49.775766+0000","last_clean":"2024-09-21T10:23:49.775766+0000","last_became_active":"2024-09-21T10:20:13.088123+0000","last_became_peered":"2024-09-21T10:20:13.088123+0000","last_unstale":"2024-09-21T10:23:49.775766+0000","last_undegraded":"2024-09-21T10:23:49.775766+0000","last_fullsized":"2024-09-21T10:23:49.775766+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:49.475499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"367'12277","reported_seq":17035,"reported_epoch":367,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:23:53.913900+0000","last_change":"2024-09-21T10:22:01.405956+0000","last_active":"2024-09-21T10:23:53.913900+0000","last_peered":"2024-09-21T10:23:53.913900+0000","last_clean":"2024-09-21T10:20:12.067444+0000","last_became_active":"2024-09-21T10:22:01.391808+0000","last_became_peered":"2024-09-21T10:22:01.391808+0000","last_unstale":"2024-09-21T10:23:53.913900+0000","last_undegraded":"2024-09-21T10:22:01.123056+0000","last_fullsized":"2024-09-21T10:23:53.913900+0000","mapping_epoch":346,"log_start":"333'5400","ondisk_log_start":"333'5400","created":305,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":6877,"log_dups_size":0,"ondisk_log_size":6877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18284544,"num_objects":279,"num_object_clones":0,"num_object_copies":837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":279,"num_objects_misplaced":279,"num_objects_unfound":0,"num_objects_dirty":279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12277,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2300,"num_bytes_recovered":150601728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[3,5,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,5","objects":279}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":622,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770652+0000","last_change":"2024-09-21T10:22:04.032207+0000","last_active":"2024-09-21T10:23:48.770652+0000","last_peered":"2024-09-21T10:23:48.770652+0000","last_clean":"2024-09-21T10:23:48.770652+0000","last_became_active":"2024-09-21T10:22:04.031881+0000","last_became_peered":"2024-09-21T10:22:04.031881+0000","last_unstale":"2024-09-21T10:23:48.770652+0000","last_undegraded":"2024-09-21T10:23:48.770652+0000","last_fullsized":"2024-09-21T10:23:48.770652+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:08.332410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:08.332410+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":645,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.773805+0000","last_change":"2024-09-21T10:21:34.876196+0000","last_active":"2024-09-21T10:23:49.773805+0000","last_peered":"2024-09-21T10:23:49.773805+0000","last_clean":"2024-09-21T10:23:49.773805+0000","last_became_active":"2024-09-21T10:21:34.868031+0000","last_became_peered":"2024-09-21T10:21:34.868031+0000","last_unstale":"2024-09-21T10:23:49.773805+0000","last_undegraded":"2024-09-21T10:23:49.773805+0000","last_fullsized":"2024-09-21T10:23:49.773805+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:58.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":990,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.781451+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:23:49.781451+0000","last_peered":"2024-09-21T10:23:49.781451+0000","last_clean":"2024-09-21T10:23:49.781451+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:23:49.781451+0000","last_undegraded":"2024-09-21T10:23:49.781451+0000","last_fullsized":"2024-09-21T10:23:49.781451+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":641,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.772692+0000","last_change":"2024-09-21T10:22:01.329400+0000","last_active":"2024-09-21T10:23:49.772692+0000","last_peered":"2024-09-21T10:23:49.772692+0000","last_clean":"2024-09-21T10:23:49.772692+0000","last_became_active":"2024-09-21T10:22:01.328887+0000","last_became_peered":"2024-09-21T10:22:01.328887+0000","last_unstale":"2024-09-21T10:23:49.772692+0000","last_undegraded":"2024-09-21T10:23:49.772692+0000","last_fullsized":"2024-09-21T10:23:49.772692+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:09.626637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"367'12160","reported_seq":21684,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:55.085664+0000","last_change":"2024-09-21T10:23:48.810067+0000","last_active":"2024-09-21T10:23:55.085664+0000","last_peered":"2024-09-21T10:23:55.085664+0000","last_clean":"2024-09-21T10:23:55.085664+0000","last_became_active":"2024-09-21T10:23:48.771862+0000","last_became_peered":"2024-09-21T10:23:48.771862+0000","last_unstale":"2024-09-21T10:23:55.085664+0000","last_undegraded":"2024-09-21T10:23:55.085664+0000","last_fullsized":"2024-09-21T10:23:55.085664+0000","mapping_epoch":365,"log_start":"317'2100","ondisk_log_start":"317'2100","created":305,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:19:21.625671+0000","last_clean_scrub_stamp":"2024-09-21T10:19:21.625671+0000","objects_scrubbed":0,"log_size":10060,"log_dups_size":0,"ondisk_log_size":10060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:35:56.641831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13697048,"num_objects":210,"num_object_clones":0,"num_object_copies":630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":210,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12160,"num_write_kb":395777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4695,"num_bytes_recovered":212533272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":610,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770339+0000","last_change":"2024-09-21T10:23:26.669475+0000","last_active":"2024-09-21T10:23:48.770339+0000","last_peered":"2024-09-21T10:23:48.770339+0000","last_clean":"2024-09-21T10:23:48.770339+0000","last_became_active":"2024-09-21T10:23:26.668318+0000","last_became_peered":"2024-09-21T10:23:26.668318+0000","last_unstale":"2024-09-21T10:23:48.770339+0000","last_undegraded":"2024-09-21T10:23:48.770339+0000","last_fullsized":"2024-09-21T10:23:48.770339+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:02.435487+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:02.435487+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":596029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":56,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.775942+0000","last_change":"2024-09-21T10:23:24.764361+0000","last_active":"2024-09-21T10:23:49.775942+0000","last_peered":"2024-09-21T10:23:49.775942+0000","last_clean":"2024-09-21T10:23:49.775942+0000","last_became_active":"2024-09-21T10:22:01.317312+0000","last_became_peered":"2024-09-21T10:22:01.317312+0000","last_unstale":"2024-09-21T10:23:49.775942+0000","last_undegraded":"2024-09-21T10:23:49.775942+0000","last_fullsized":"2024-09-21T10:23:49.775942+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:32:39.440581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":653,"reported_epoch":367,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:23:49.775135+0000","last_change":"2024-09-21T10:21:34.866048+0000","last_active":"2024-09-21T10:23:49.775135+0000","last_peered":"2024-09-21T10:23:49.775135+0000","last_clean":"2024-09-21T10:21:17.726080+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:23:49.775135+0000","last_undegraded":"2024-09-21T10:21:34.856315+0000","last_fullsized":"2024-09-21T10:23:49.775135+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":339,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819172+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:23:48.819172+0000","last_peered":"2024-09-21T10:23:48.819172+0000","last_clean":"2024-09-21T10:23:48.819172+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:23:48.819172+0000","last_undegraded":"2024-09-21T10:23:48.819172+0000","last_fullsized":"2024-09-21T10:23:48.819172+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":364,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819678+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:23:48.819678+0000","last_peered":"2024-09-21T10:23:48.819678+0000","last_clean":"2024-09-21T10:23:48.819678+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:23:48.819678+0000","last_undegraded":"2024-09-21T10:23:48.819678+0000","last_fullsized":"2024-09-21T10:23:48.819678+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":73,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819279+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:23:48.819279+0000","last_peered":"2024-09-21T10:23:48.819279+0000","last_clean":"2024-09-21T10:23:48.819279+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:23:48.819279+0000","last_undegraded":"2024-09-21T10:23:48.819279+0000","last_fullsized":"2024-09-21T10:23:48.819279+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768773+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:23:48.768773+0000","last_peered":"2024-09-21T10:23:48.768773+0000","last_clean":"2024-09-21T10:23:48.768773+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:23:48.768773+0000","last_undegraded":"2024-09-21T10:23:48.768773+0000","last_fullsized":"2024-09-21T10:23:48.768773+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":45,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.770493+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:23:48.770493+0000","last_peered":"2024-09-21T10:23:48.770493+0000","last_clean":"2024-09-21T10:23:48.770493+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:23:48.770493+0000","last_undegraded":"2024-09-21T10:23:48.770493+0000","last_fullsized":"2024-09-21T10:23:48.770493+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":74,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768627+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:23:48.768627+0000","last_peered":"2024-09-21T10:23:48.768627+0000","last_clean":"2024-09-21T10:23:48.768627+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:23:48.768627+0000","last_undegraded":"2024-09-21T10:23:48.768627+0000","last_fullsized":"2024-09-21T10:23:48.768627+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":564,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.772524+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:23:49.772524+0000","last_peered":"2024-09-21T10:23:49.772524+0000","last_clean":"2024-09-21T10:23:49.772524+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:23:49.772524+0000","last_undegraded":"2024-09-21T10:23:49.772524+0000","last_fullsized":"2024-09-21T10:23:49.772524+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":220,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.776173+0000","last_change":"2024-09-21T10:20:13.089473+0000","last_active":"2024-09-21T10:23:49.776173+0000","last_peered":"2024-09-21T10:23:49.776173+0000","last_clean":"2024-09-21T10:23:49.776173+0000","last_became_active":"2024-09-21T10:20:13.089111+0000","last_became_peered":"2024-09-21T10:20:13.089111+0000","last_unstale":"2024-09-21T10:23:49.776173+0000","last_undegraded":"2024-09-21T10:23:49.776173+0000","last_fullsized":"2024-09-21T10:23:49.776173+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:06.603284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":624,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.772575+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:23:49.772575+0000","last_peered":"2024-09-21T10:23:49.772575+0000","last_clean":"2024-09-21T10:23:49.772575+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:23:49.772575+0000","last_undegraded":"2024-09-21T10:23:49.772575+0000","last_fullsized":"2024-09-21T10:23:49.772575+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":381,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.769451+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:23:48.769451+0000","last_peered":"2024-09-21T10:23:48.769451+0000","last_clean":"2024-09-21T10:23:48.769451+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:23:48.769451+0000","last_undegraded":"2024-09-21T10:23:48.769451+0000","last_fullsized":"2024-09-21T10:23:48.769451+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":641,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.772733+0000","last_change":"2024-09-21T10:22:01.329690+0000","last_active":"2024-09-21T10:23:49.772733+0000","last_peered":"2024-09-21T10:23:49.772733+0000","last_clean":"2024-09-21T10:23:49.772733+0000","last_became_active":"2024-09-21T10:22:01.329033+0000","last_became_peered":"2024-09-21T10:22:01.329033+0000","last_unstale":"2024-09-21T10:23:49.772733+0000","last_undegraded":"2024-09-21T10:23:49.772733+0000","last_fullsized":"2024-09-21T10:23:49.772733+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:24.559524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":223,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.776437+0000","last_change":"2024-09-21T10:21:34.870704+0000","last_active":"2024-09-21T10:23:49.776437+0000","last_peered":"2024-09-21T10:23:49.776437+0000","last_clean":"2024-09-21T10:23:49.776437+0000","last_became_active":"2024-09-21T10:21:34.869835+0000","last_became_peered":"2024-09-21T10:21:34.869835+0000","last_unstale":"2024-09-21T10:23:49.776437+0000","last_undegraded":"2024-09-21T10:23:49.776437+0000","last_fullsized":"2024-09-21T10:23:49.776437+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:34.959915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":339,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.780886+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:23:49.780886+0000","last_peered":"2024-09-21T10:23:49.780886+0000","last_clean":"2024-09-21T10:23:49.780886+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:23:49.780886+0000","last_undegraded":"2024-09-21T10:23:49.780886+0000","last_fullsized":"2024-09-21T10:23:49.780886+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":631,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.773259+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:23:49.773259+0000","last_peered":"2024-09-21T10:23:49.773259+0000","last_clean":"2024-09-21T10:23:49.773259+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:23:49.773259+0000","last_undegraded":"2024-09-21T10:23:49.773259+0000","last_fullsized":"2024-09-21T10:23:49.773259+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":371,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819328+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:23:48.819328+0000","last_peered":"2024-09-21T10:23:48.819328+0000","last_clean":"2024-09-21T10:23:48.819328+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:23:48.819328+0000","last_undegraded":"2024-09-21T10:23:48.819328+0000","last_fullsized":"2024-09-21T10:23:48.819328+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":630,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.814786+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:23:49.814786+0000","last_peered":"2024-09-21T10:23:49.814786+0000","last_clean":"2024-09-21T10:23:49.814786+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:23:49.814786+0000","last_undegraded":"2024-09-21T10:23:49.814786+0000","last_fullsized":"2024-09-21T10:23:49.814786+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":52,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.775731+0000","last_change":"2024-09-21T10:22:01.318549+0000","last_active":"2024-09-21T10:23:49.775731+0000","last_peered":"2024-09-21T10:23:49.775731+0000","last_clean":"2024-09-21T10:23:49.775731+0000","last_became_active":"2024-09-21T10:22:01.318071+0000","last_became_peered":"2024-09-21T10:22:01.318071+0000","last_unstale":"2024-09-21T10:23:49.775731+0000","last_undegraded":"2024-09-21T10:23:49.775731+0000","last_fullsized":"2024-09-21T10:23:49.775731+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:49.061492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":63,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819427+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:23:48.819427+0000","last_peered":"2024-09-21T10:23:48.819427+0000","last_clean":"2024-09-21T10:23:48.819427+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:23:48.819427+0000","last_undegraded":"2024-09-21T10:23:48.819427+0000","last_fullsized":"2024-09-21T10:23:48.819427+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":306,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768195+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:23:48.768195+0000","last_peered":"2024-09-21T10:23:48.768195+0000","last_clean":"2024-09-21T10:23:48.768195+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:23:48.768195+0000","last_undegraded":"2024-09-21T10:23:48.768195+0000","last_fullsized":"2024-09-21T10:23:48.768195+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":63,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.819085+0000","last_change":"2024-09-21T10:19:50.588363+0000","last_active":"2024-09-21T10:23:48.819085+0000","last_peered":"2024-09-21T10:23:48.819085+0000","last_clean":"2024-09-21T10:23:48.819085+0000","last_became_active":"2024-09-21T10:19:50.588107+0000","last_became_peered":"2024-09-21T10:19:50.588107+0000","last_unstale":"2024-09-21T10:23:48.819085+0000","last_undegraded":"2024-09-21T10:23:48.819085+0000","last_fullsized":"2024-09-21T10:23:48.819085+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:05.698018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":284,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.768626+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:23:48.768626+0000","last_peered":"2024-09-21T10:23:48.768626+0000","last_clean":"2024-09-21T10:23:48.768626+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:23:48.768626+0000","last_undegraded":"2024-09-21T10:23:48.768626+0000","last_fullsized":"2024-09-21T10:23:48.768626+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":644,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.772814+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:23:49.772814+0000","last_peered":"2024-09-21T10:23:49.772814+0000","last_clean":"2024-09-21T10:23:49.772814+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:23:49.772814+0000","last_undegraded":"2024-09-21T10:23:49.772814+0000","last_fullsized":"2024-09-21T10:23:49.772814+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":606,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.773485+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:23:49.773485+0000","last_peered":"2024-09-21T10:23:49.773485+0000","last_clean":"2024-09-21T10:23:49.773485+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:23:49.773485+0000","last_undegraded":"2024-09-21T10:23:49.773485+0000","last_fullsized":"2024-09-21T10:23:49.773485+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":63,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-21T10:23:48.818892+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:23:48.818892+0000","last_peered":"2024-09-21T10:23:48.818892+0000","last_clean":"2024-09-21T10:23:48.818892+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:23:48.818892+0000","last_undegraded":"2024-09-21T10:23:48.818892+0000","last_fullsized":"2024-09-21T10:23:48.818892+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":58,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-21T10:23:49.776090+0000","last_change":"2024-09-21T10:23:24.753979+0000","last_active":"2024-09-21T10:23:49.776090+0000","last_peered":"2024-09-21T10:23:49.776090+0000","last_clean":"2024-09-21T10:23:49.776090+0000","last_became_active":"2024-09-21T10:22:01.316569+0000","last_became_peered":"2024-09-21T10:22:01.316569+0000","last_unstale":"2024-09-21T10:23:49.776090+0000","last_undegraded":"2024-09-21T10:23:49.776090+0000","last_fullsized":"2024-09-21T10:23:49.776090+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:35:44.904752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":227409944,"num_objects":3471,"num_object_clones":0,"num_object_copies":10413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11217,"num_objects_misplaced":279,"num_objects_unfound":0,"num_objects_dirty":3471,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":196215,"num_write_kb":6389889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27748,"num_bytes_recovered":1433141272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1454260224,"data_stored":1454243936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110986,"ondisk_log_size":110986,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":6328,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24225,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":391,"num_read_kb":833,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3697556,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1640,"ondisk_log_size":1640,"up":33,"acting":33,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234411,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":679456,"kb_used_data":74904,"kb_used_omap":1,"kb_used_meta":604542,"kb_avail":93065696,"statfs":{"total":95995035648,"available":95299272704,"internally_reserved":0,"allocated":76701696,"data_stored":74782433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":619051467},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496049,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1199580,"kb_used_data":572820,"kb_used_omap":1,"kb_used_meta":626686,"kb_avail":92545572,"statfs":{"total":95995035648,"available":94766665728,"internally_reserved":0,"allocated":586567680,"data_stored":584652717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":641726919},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403949,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":412496,"kb_used_data":136072,"kb_used_omap":7,"kb_used_meta":276408,"kb_avail":93332656,"statfs":{"total":95995035648,"available":95572639744,"internally_reserved":0,"allocated":139337728,"data_stored":137430335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":283042712},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,4,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019322,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":695128,"kb_used_data":92112,"kb_used_omap":1,"kb_used_meta":602878,"kb_avail":93050024,"statfs":{"total":95995035648,"available":95283224576,"internally_reserved":0,"allocated":94322688,"data_stored":92411310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":617347526},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435506,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1182084,"kb_used_data":392380,"kb_used_omap":5,"kb_used_meta":789690,"kb_avail":92563068,"statfs":{"total":95995035648,"available":94784581632,"internally_reserved":0,"allocated":401797120,"data_stored":399901422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":808642600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599033,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":817480,"kb_used_data":113856,"kb_used_omap":7,"kb_used_meta":703608,"kb_avail":92927672,"statfs":{"total":95995035648,"available":95157936128,"internally_reserved":0,"allocated":116588544,"data_stored":114688992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":720495053},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762560,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":701640,"kb_used_data":96576,"kb_used_omap":7,"kb_used_meta":605048,"kb_avail":93043512,"statfs":{"total":95995035648,"available":95276556288,"internally_reserved":0,"allocated":98893824,"data_stored":96994252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":619569405},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926087,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":547724,"kb_used_data":64836,"kb_used_omap":7,"kb_used_meta":482872,"kb_avail":93197428,"statfs":{"total":95995035648,"available":95434166272,"internally_reserved":0,"allocated":66392064,"data_stored":64489232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":494461118},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":44761088,"data_stored":44761088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":88473600,"data_stored":88473600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":99028992,"data_stored":99024920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":378339328,"data_stored":378339328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":88014848,"data_stored":88014848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":125702144,"data_stored":125698072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":570167296,"data_stored":570163224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":59772928,"data_stored":59768856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:24:01.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:24:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.393 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:01.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:01.494 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:01.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:01.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:01.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:01.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.642 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:01.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:01.744 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:01.766 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:01.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:01.868 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:01.892 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:01.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:01.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:02.016 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:01 smithi045 ceph-mon[95976]: 1.a scrub starts 2024-09-21T10:24:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:01 smithi045 ceph-mon[95976]: pgmap v1112: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 28 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 10607/1185 objects degraded (895.105%); 15/1185 objects misplaced (1.266%); 0 B/s, 56 objects/s recovering 2024-09-21T10:24:02.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:01 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10607/1185 objects degraded (895.105%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:24:02.118 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:02.125 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:01 smithi032 ceph-mon[101299]: 1.a scrub starts 2024-09-21T10:24:02.126 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:01 smithi032 ceph-mon[101299]: pgmap v1112: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 28 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 10607/1185 objects degraded (895.105%); 15/1185 objects misplaced (1.266%); 0 B/s, 56 objects/s recovering 2024-09-21T10:24:02.126 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:01 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10607/1185 objects degraded (895.105%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:24:02.126 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:01 smithi032 ceph-mon[105748]: 1.a scrub starts 2024-09-21T10:24:02.126 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:01 smithi032 ceph-mon[105748]: pgmap v1112: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 28 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 10607/1185 objects degraded (895.105%); 15/1185 objects misplaced (1.266%); 0 B/s, 56 objects/s recovering 2024-09-21T10:24:02.126 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:01 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10607/1185 objects degraded (895.105%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:24:02.142 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:02.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:02.212 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:24:02.243 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:02.266 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:02.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:02.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:02.392 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:02.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:02.397 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:24:02.397 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:24:02.398 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:24:02.398 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:24:02.403 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:02.419 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:24:02.419 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:24:02.440 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:24:02.441 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:24:02.457 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:24:02.458 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:24:02.461 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:02.475 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:24:02.475 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:24:02.493 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:24:02.493 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:24:02.493 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:02.511 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:24:02.511 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:24:02.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:02.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:02.619 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:02.641 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:02.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:02.743 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:02.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:02.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:02.783 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:24:02.867 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:02.890 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:02.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:02.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:02 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1135028307' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:02.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:02 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1135028307' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:02.992 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:03.015 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:03.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:03.116 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:03.142 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:03.244 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:03.266 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:03.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:03.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:02 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1135028307' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:03.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:03.391 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:03.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:03.492 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:03.515 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:03.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:03.617 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:03.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.642 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:03.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:03.743 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:03.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:03.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:03.867 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:03.889 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:03.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:03.991 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:04.013 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:04.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:04.115 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:04.137 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:04.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:03 smithi032 ceph-mon[105748]: pgmap v1113: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 9.8 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 10607/333 objects degraded (3185.285%); 15/333 objects misplaced (4.505%); 0 B/s, 56 objects/s recovering 2024-09-21T10:24:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:04.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:03 smithi032 ceph-mon[101299]: pgmap v1113: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 9.8 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 10607/333 objects degraded (3185.285%); 15/333 objects misplaced (4.505%); 0 B/s, 56 objects/s recovering 2024-09-21T10:24:04.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:04.239 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:04.262 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:04.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:03 smithi045 ceph-mon[95976]: pgmap v1113: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 9.8 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 10607/333 objects degraded (3185.285%); 15/333 objects misplaced (4.505%); 0 B/s, 56 objects/s recovering 2024-09-21T10:24:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:04.364 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:04.387 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:04.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:04.414 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-09-21T10:24:04.414 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-09-21T10:24:04.489 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:04.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:04.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:04.612 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:04.635 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:04.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:04.759 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:04.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:04.861 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:04.883 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:04.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:05.003 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:04 smithi045 ceph-mon[95976]: 1.0 scrub starts 2024-09-21T10:24:05.004 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:05.004 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:05.007 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:05.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:05.108 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.134 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:05.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:05.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:04 smithi032 ceph-mon[101299]: 1.0 scrub starts 2024-09-21T10:24:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:04 smithi032 ceph-mon[105748]: 1.0 scrub starts 2024-09-21T10:24:05.234 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:05.257 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:05.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:05.358 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:05.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.382 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:05.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:05.483 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:05.505 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:05.607 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:05.629 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:05.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:05.731 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:05.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:05.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:05.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:05.879 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:05.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:05.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:05.994 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:05 smithi045 ceph-mon[95976]: pgmap v1114: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 10592/84 objects degraded (12609.524%); 0 B/s, 56 objects/s recovering 2024-09-21T10:24:06.004 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:06.017 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:24:06.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:06.130 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:05 smithi032 ceph-mon[101299]: pgmap v1114: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 10592/84 objects degraded (12609.524%); 0 B/s, 56 objects/s recovering 2024-09-21T10:24:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:05 smithi032 ceph-mon[105748]: pgmap v1114: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 10592/84 objects degraded (12609.524%); 0 B/s, 56 objects/s recovering 2024-09-21T10:24:06.232 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:06.232 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:06.254 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:06.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:06.356 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:06.378 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:06.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:06.480 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:06.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.503 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:06.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:06.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:06.627 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:06.729 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:06.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:06.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:06.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:06.876 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:06.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:06.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:07.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.003 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:07.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:07.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:06 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10592/84 objects degraded (12609.524%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:24:07.103 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:07.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.126 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:07.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:07.187 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:06 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10592/84 objects degraded (12609.524%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:24:07.187 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:06 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10592/84 objects degraded (12609.524%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:24:07.228 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:07.250 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:07.351 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:07.374 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:07.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:07.476 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:07.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:07.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:07.600 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:07.622 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:07.726 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:07.749 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:07.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:07.850 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:07.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:07.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:07.975 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:07.997 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:08.099 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:08.116 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:07 smithi045 ceph-mon[95976]: 1.3 scrub starts 2024-09-21T10:24:08.116 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:07 smithi045 ceph-mon[95976]: pgmap v1115: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 9439/84 objects degraded (11236.905%); 0 B/s, 151 objects/s recovering 2024-09-21T10:24:08.121 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:08.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:07 smithi032 ceph-mon[101299]: 1.3 scrub starts 2024-09-21T10:24:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:07 smithi032 ceph-mon[101299]: pgmap v1115: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 9439/84 objects degraded (11236.905%); 0 B/s, 151 objects/s recovering 2024-09-21T10:24:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:07 smithi032 ceph-mon[105748]: 1.3 scrub starts 2024-09-21T10:24:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:07 smithi032 ceph-mon[105748]: pgmap v1115: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 9439/84 objects degraded (11236.905%); 0 B/s, 151 objects/s recovering 2024-09-21T10:24:08.223 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:08.245 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:08.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:08.347 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:08.370 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:08.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:08.399 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:08.400 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:08.472 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:08.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:08.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:08.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:08.628 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:08.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:08.730 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:08.753 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:08.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:08.877 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:08.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:08.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:09.002 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:09.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:09.104 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:09.127 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:09.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:09.228 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.253 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:09.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:09.354 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:09.376 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:09.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:09.450 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:24:09.478 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:09.517 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:09.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:09.536 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:09.618 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:09.641 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:09.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:09.661 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:24:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:24:09] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.51.0" 2024-09-21T10:24:09.672 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:09.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:09.767 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:09.808 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:24:09.809 INFO:teuthology.orchestra.run.smithi032.stdout:{"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 9439/84 objects degraded (11236.905%), 6 pgs degraded, 5 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1540,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":368,"num_osds":8,"num_up_osds":8,"osd_up_since":1726914012,"num_in_osds":8,"osd_in_since":1726913989,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":90},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+clean+scrubbing","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":28,"data_bytes":4790245,"bytes_used":5630078976,"bytes_avail":762330206208,"bytes_total":767960285184,"degraded_objects":9439,"degraded_total":84,"degraded_ratio":112.36904761904762,"recovering_objects_per_sec":151,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":1517,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":347},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi032.fdolcm","status":"up:active","gid":14655}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.32:8443/","prometheus":"http://172.21.15.32:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-21T10:21:12.493782+0000","services":{}},"progress_events":{}} 2024-09-21T10:24:09.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:09.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:09 smithi032 ceph-mon[101299]: pgmap v1116: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 9439/84 objects degraded (11236.905%); 0 B/s, 151 objects/s recovering 2024-09-21T10:24:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:09 smithi032 ceph-mon[101299]: 1.7 scrub starts 2024-09-21T10:24:09.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3326629643' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:24:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:09 smithi032 ceph-mon[105748]: pgmap v1116: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 9439/84 objects degraded (11236.905%); 0 B/s, 151 objects/s recovering 2024-09-21T10:24:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:09 smithi032 ceph-mon[105748]: 1.7 scrub starts 2024-09-21T10:24:09.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3326629643' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:24:09.994 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:10.017 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:10.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:10.119 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:10.141 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:10.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:10.232 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:09 smithi045 ceph-mon[95976]: pgmap v1116: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 9439/84 objects degraded (11236.905%); 0 B/s, 151 objects/s recovering 2024-09-21T10:24:10.233 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:09 smithi045 ceph-mon[95976]: 1.7 scrub starts 2024-09-21T10:24:10.233 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3326629643' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:24:10.243 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:10.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.266 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:10.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:10.367 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:10.390 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:10.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:10.450 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:24:10.493 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:10.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:10.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:10.618 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:10.641 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:10.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:10.765 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:10.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:10.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:10.990 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:11.013 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:11.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:11.115 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:11.137 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:11.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:11.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:10 smithi032 ceph-mon[101299]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T10:24:11.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:10 smithi032 ceph-mon[105748]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T10:24:11.240 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:11.265 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:11.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:10 smithi045 ceph-mon[95976]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T10:24:11.367 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:11.389 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:11.491 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:11.514 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:11.616 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:11.639 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:11.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:11.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:11.763 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:11.865 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:11.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:11.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:11.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:12.011 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:11 smithi045 ceph-mon[95976]: pgmap v1118: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 7801/84 objects degraded (9286.905%); 0 B/s, 279 objects/s recovering 2024-09-21T10:24:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:11 smithi045 ceph-mon[95976]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T10:24:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:11 smithi045 ceph-mon[95976]: 1.9 scrub starts 2024-09-21T10:24:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:11 smithi045 ceph-mon[95976]: 1.9 scrub ok 2024-09-21T10:24:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:11 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 7801/84 objects degraded (9286.905%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:24:12.114 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:12.136 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:12.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[101299]: pgmap v1118: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 7801/84 objects degraded (9286.905%); 0 B/s, 279 objects/s recovering 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[101299]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[101299]: 1.9 scrub starts 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[101299]: 1.9 scrub ok 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 7801/84 objects degraded (9286.905%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[105748]: pgmap v1118: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 7801/84 objects degraded (9286.905%); 0 B/s, 279 objects/s recovering 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[105748]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[105748]: 1.9 scrub starts 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[105748]: 1.9 scrub ok 2024-09-21T10:24:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:11 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 7801/84 objects degraded (9286.905%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:24:12.238 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:12.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:12.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:12.334 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:12.334 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:12.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:12.384 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:12.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:12.486 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:12.508 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:12.610 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:12.633 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:12.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:12.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:12.757 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:12.859 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:12.882 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:12.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:12.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:13.007 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:13.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:13.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:12 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:24:13.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:24:13.109 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:13.131 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:13.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:12 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:24:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:12 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:24:13.234 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:13.257 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:13.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:13.358 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:13.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:13.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:13.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:13.505 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:13.607 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:13.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:13.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:13.734 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:13.756 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:13.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:13.777 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:13.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:13.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:13.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:14.005 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:14.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:14.021 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:13 smithi045 ceph-mon[95976]: pgmap v1120: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7801/84 objects degraded (9286.905%); 0 B/s, 348 objects/s recovering 2024-09-21T10:24:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:13 smithi045 ceph-mon[95976]: 1.6 scrub starts 2024-09-21T10:24:14.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:13 smithi045 ceph-mon[95976]: 1.6 scrub ok 2024-09-21T10:24:14.107 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:14.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:14.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:13 smithi032 ceph-mon[105748]: pgmap v1120: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7801/84 objects degraded (9286.905%); 0 B/s, 348 objects/s recovering 2024-09-21T10:24:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:13 smithi032 ceph-mon[105748]: 1.6 scrub starts 2024-09-21T10:24:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:13 smithi032 ceph-mon[105748]: 1.6 scrub ok 2024-09-21T10:24:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:13 smithi032 ceph-mon[101299]: pgmap v1120: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7801/84 objects degraded (9286.905%); 0 B/s, 348 objects/s recovering 2024-09-21T10:24:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:13 smithi032 ceph-mon[101299]: 1.6 scrub starts 2024-09-21T10:24:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:13 smithi032 ceph-mon[101299]: 1.6 scrub ok 2024-09-21T10:24:14.229 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:24:14.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:24:14.231 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:14.295 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:14.397 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:14.419 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:14.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:14.520 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:14.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:14.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.667 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:14.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:14.768 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:14.791 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:14.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:14.893 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:14.917 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:14.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:15.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:15.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.044 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:15.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:15.145 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:15.147 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:14 smithi032 ceph-mon[105748]: 1.1 scrub starts 2024-09-21T10:24:15.147 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:14 smithi032 ceph-mon[105748]: 1.1 scrub ok 2024-09-21T10:24:15.147 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:14 smithi032 ceph-mon[101299]: 1.1 scrub starts 2024-09-21T10:24:15.147 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:14 smithi032 ceph-mon[101299]: 1.1 scrub ok 2024-09-21T10:24:15.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.168 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:15.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:15.224 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:14 smithi045 ceph-mon[95976]: 1.1 scrub starts 2024-09-21T10:24:15.225 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:14 smithi045 ceph-mon[95976]: 1.1 scrub ok 2024-09-21T10:24:15.269 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:15.291 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:15.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:15.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:15.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:15.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:15.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:15.540 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:15.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:15.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:15.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:15.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:15.767 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:15.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:15.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:15.937 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:15.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:16.039 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:16.061 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:16.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:16.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:16.167 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:15 smithi045 ceph-mon[95976]: pgmap v1121: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7801/84 objects degraded (9286.905%); 0 B/s, 204 objects/s recovering 2024-09-21T10:24:16.167 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:15 smithi045 ceph-mon[95976]: 1.2 scrub starts 2024-09-21T10:24:16.167 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:15 smithi045 ceph-mon[95976]: 1.2 scrub ok 2024-09-21T10:24:16.167 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:15 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/88237448' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:24:16.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:16.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:15 smithi032 ceph-mon[101299]: pgmap v1121: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7801/84 objects degraded (9286.905%); 0 B/s, 204 objects/s recovering 2024-09-21T10:24:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:15 smithi032 ceph-mon[101299]: 1.2 scrub starts 2024-09-21T10:24:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:15 smithi032 ceph-mon[101299]: 1.2 scrub ok 2024-09-21T10:24:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:15 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/88237448' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:24:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:15 smithi032 ceph-mon[105748]: pgmap v1121: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7801/84 objects degraded (9286.905%); 0 B/s, 204 objects/s recovering 2024-09-21T10:24:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:15 smithi032 ceph-mon[105748]: 1.2 scrub starts 2024-09-21T10:24:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:15 smithi032 ceph-mon[105748]: 1.2 scrub ok 2024-09-21T10:24:16.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:15 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/88237448' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:24:16.289 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:16.311 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:16.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:16.413 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:16.416 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:16.417 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:16.435 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:16.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:16.489 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_3' removed 2024-09-21T10:24:16.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:16.560 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:16.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:16.661 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:16.685 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:16.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:16.787 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:16.810 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:16.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:16.912 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:16.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:16.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:17.035 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:17.043 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:16 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/88237448' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:24:17.044 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:16 smithi032 ceph-mon[105748]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T10:24:17.044 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:16 smithi032 ceph-mon[105748]: osdmap e372: 8 total, 8 up, 8 in 2024-09-21T10:24:17.044 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:16 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/88237448' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:24:17.044 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:16 smithi032 ceph-mon[101299]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T10:24:17.044 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:16 smithi032 ceph-mon[101299]: osdmap e372: 8 total, 8 up, 8 in 2024-09-21T10:24:17.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.058 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:17.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:17.064 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:24:17.065 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:24:17.065 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-09-21T10:24:17.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_4 16 2024-09-21T10:24:17.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:24:17.138 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:16 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/88237448' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:24:17.138 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:16 smithi045 ceph-mon[95976]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T10:24:17.138 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:16 smithi045 ceph-mon[95976]: osdmap e372: 8 total, 8 up, 8 in 2024-09-21T10:24:17.159 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:17.182 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:17.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:17.284 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:17.306 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:17.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:17.380 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:17.382 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:17.408 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:17.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:17.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:17.555 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:17.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:17.657 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:17.658 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:17.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:17.780 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:17.802 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:17.808 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:24:17.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:17.927 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:17.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:17.957 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:18.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:17 smithi045 ceph-mon[95976]: pgmap v1123: 83 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 18/81 objects degraded (22.222%) 2024-09-21T10:24:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 18/81 objects degraded (22.222%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:24:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:17 smithi045 ceph-mon[95976]: 1.5 scrub starts 2024-09-21T10:24:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:17 smithi045 ceph-mon[95976]: 1.5 scrub ok 2024-09-21T10:24:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:17 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3890969018' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:17 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:24:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:17 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3890969018' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:24:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:17 smithi045 ceph-mon[95976]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T10:24:18.051 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:18.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[101299]: pgmap v1123: 83 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 18/81 objects degraded (22.222%) 2024-09-21T10:24:18.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 18/81 objects degraded (22.222%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:24:18.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[101299]: 1.5 scrub starts 2024-09-21T10:24:18.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[101299]: 1.5 scrub ok 2024-09-21T10:24:18.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3890969018' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:18.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:24:18.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3890969018' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:24:18.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[101299]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T10:24:18.116 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[105748]: pgmap v1123: 83 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 18/81 objects degraded (22.222%) 2024-09-21T10:24:18.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 18/81 objects degraded (22.222%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:24:18.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[105748]: 1.5 scrub starts 2024-09-21T10:24:18.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[105748]: 1.5 scrub ok 2024-09-21T10:24:18.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3890969018' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:18.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:24:18.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3890969018' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:24:18.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:17 smithi032 ceph-mon[105748]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T10:24:18.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:18.180 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:18.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:18.281 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:18.305 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:18.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:18.407 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:18.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:18.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:18.532 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:18.554 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:18.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:18.655 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:18.677 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:18.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:18.779 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:18.801 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:18.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:18.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:18.926 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:19.027 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:19.050 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:19.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:19.152 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:19.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:19.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:18 smithi032 ceph-mon[101299]: 1.4 deep-scrub starts 2024-09-21T10:24:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:18 smithi032 ceph-mon[101299]: 1.4 deep-scrub ok 2024-09-21T10:24:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:18 smithi032 ceph-mon[105748]: 1.4 deep-scrub starts 2024-09-21T10:24:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:18 smithi032 ceph-mon[105748]: 1.4 deep-scrub ok 2024-09-21T10:24:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:19.277 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:19.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:18 smithi045 ceph-mon[95976]: 1.4 deep-scrub starts 2024-09-21T10:24:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:18 smithi045 ceph-mon[95976]: 1.4 deep-scrub ok 2024-09-21T10:24:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:19.300 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:19.402 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:19.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.426 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:19.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:19.527 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:19.549 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:19.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:19.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:19.674 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:19.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:19.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:24:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:24:19] "GET /metrics HTTP/1.1" 200 34148 "" "Prometheus/2.51.0" 2024-09-21T10:24:19.776 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:19.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:19.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:19.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:19.922 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:19.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:20.024 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:20.048 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:20.149 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:20.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:20.177 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:19 smithi045 ceph-mon[95976]: pgmap v1126: 83 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 18/81 objects degraded (22.222%) 2024-09-21T10:24:20.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:19 smithi032 ceph-mon[105748]: pgmap v1126: 83 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 18/81 objects degraded (22.222%) 2024-09-21T10:24:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:19 smithi032 ceph-mon[101299]: pgmap v1126: 83 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 18/81 objects degraded (22.222%) 2024-09-21T10:24:20.274 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:20.298 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:20.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:20.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:20.424 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:20.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:20.526 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:20.547 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:20.649 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:20.672 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:20.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:20.774 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:20.801 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:20.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:20.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:20.929 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:21.020 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:21.020 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:21.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:21.057 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:21.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:21.159 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:21.182 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:21.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:21.284 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:21.311 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:21.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:21.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:24:21.413 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:21.437 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:21.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:21.561 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:21.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:21.663 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:21.685 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:21 smithi032 ceph-mon[101299]: pgmap v1127: 83 pgs: 83 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:24:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-21T10:24:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:21 smithi032 ceph-mon[105748]: pgmap v1127: 83 pgs: 83 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:24:21.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-21T10:24:21.722 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:24:21.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:21.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:21 smithi045 ceph-mon[95976]: pgmap v1127: 83 pgs: 83 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:24:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-21T10:24:21.787 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:21.811 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:21.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:21.914 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:21.938 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:21.975 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:22.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:22.064 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:22.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:22.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:22.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.190 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:22.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:22.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:22.313 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:22.415 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:22.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:22.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:22.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:22.563 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:22.665 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:22.688 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:22.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:22.751 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[105748]: 1.8 deep-scrub starts 2024-09-21T10:24:22.751 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[105748]: 1.8 deep-scrub ok 2024-09-21T10:24:22.751 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[105748]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/81 objects degraded (22.222%), 2 pgs degraded, 1 pg undersized) 2024-09-21T10:24:22.751 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:24:22.751 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-21T10:24:22.751 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[105748]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T10:24:22.751 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[105748]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T10:24:22.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[101299]: 1.8 deep-scrub starts 2024-09-21T10:24:22.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[101299]: 1.8 deep-scrub ok 2024-09-21T10:24:22.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[101299]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/81 objects degraded (22.222%), 2 pgs degraded, 1 pg undersized) 2024-09-21T10:24:22.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:24:22.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-21T10:24:22.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[101299]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T10:24:22.752 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:22 smithi032 ceph-mon[101299]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T10:24:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:22 smithi045 ceph-mon[95976]: 1.8 deep-scrub starts 2024-09-21T10:24:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:22 smithi045 ceph-mon[95976]: 1.8 deep-scrub ok 2024-09-21T10:24:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:22 smithi045 ceph-mon[95976]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/81 objects degraded (22.222%), 2 pgs degraded, 1 pg undersized) 2024-09-21T10:24:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:22 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:24:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:22 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-21T10:24:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:22 smithi045 ceph-mon[95976]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T10:24:22.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:22 smithi045 ceph-mon[95976]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T10:24:22.789 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:22.814 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:22.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:22.938 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:22.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:23.040 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:23.064 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:23.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:23.165 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:23.189 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:23.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:23.291 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:23.314 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:23.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:23.415 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:23.438 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:23.540 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:23.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:23.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:23.665 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:23.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:23.697 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:23.697 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:23.789 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:23.811 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:23.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:23.913 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:23.936 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:23.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:24.037 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:24.042 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:23 smithi032 ceph-mon[101299]: pgmap v1130: 83 pgs: 83 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2024-09-21T10:24:24.042 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:23 smithi032 ceph-mon[101299]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T10:24:24.042 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:23 smithi032 ceph-mon[105748]: pgmap v1130: 83 pgs: 83 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2024-09-21T10:24:24.042 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:23 smithi032 ceph-mon[105748]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T10:24:24.060 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:24.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:24.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:23 smithi045 ceph-mon[95976]: pgmap v1130: 83 pgs: 83 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2024-09-21T10:24:24.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:23 smithi045 ceph-mon[95976]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T10:24:24.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:24.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:24.286 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:24.308 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:24.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:24.389 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:24:24.411 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:24.433 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:24.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:24.534 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:24.558 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:24.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:24.593 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:24.659 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:24.681 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:24.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:24.705 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:24:24.706 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:24:24.783 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:24.806 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:24.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:24.819 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_4' created 2024-09-21T10:24:24.908 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:24.930 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:24.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:24 smithi032 ceph-mon[101299]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T10:24:24.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:24 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/796444551' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:24:24.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:24 smithi032 ceph-mon[105748]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T10:24:24.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:24 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/796444551' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:24:24.979 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:24 smithi045 ceph-mon[95976]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T10:24:24.979 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:24 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/796444551' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:24:25.033 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:25.055 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:25.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:25.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:25.180 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:25.282 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:25.306 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:25.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:25.407 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:25.483 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:25.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:25.585 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:25.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:25.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:25.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:25.734 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:25.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:25.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-09-21T10:24:25.836 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:25.863 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:25 smithi045 ceph-mon[95976]: pgmap v1133: 83 pgs: 1 peering, 82 active+clean; 4.3 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 225 KiB/s, 0 objects/s recovering 2024-09-21T10:24:25.863 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:25 smithi045 ceph-mon[95976]: from='client.45821 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:25.863 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/796444551' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-21T10:24:25.863 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:25 smithi045 ceph-mon[95976]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T10:24:25.863 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:25 smithi045 ceph-mon[95976]: 1.0 scrub starts 2024-09-21T10:24:25.863 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:25 smithi045 ceph-mon[95976]: 1.0 scrub ok 2024-09-21T10:24:25.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:25.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:25.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:26.018 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:26.120 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:26.121 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:26.122 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:26.142 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:26.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[101299]: pgmap v1133: 83 pgs: 1 peering, 82 active+clean; 4.3 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 225 KiB/s, 0 objects/s recovering 2024-09-21T10:24:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[101299]: from='client.45821 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/796444551' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-21T10:24:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[101299]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T10:24:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[101299]: 1.0 scrub starts 2024-09-21T10:24:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[101299]: 1.0 scrub ok 2024-09-21T10:24:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[105748]: pgmap v1133: 83 pgs: 1 peering, 82 active+clean; 4.3 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 225 KiB/s, 0 objects/s recovering 2024-09-21T10:24:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[105748]: from='client.45821 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/796444551' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-21T10:24:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[105748]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T10:24:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[105748]: 1.0 scrub starts 2024-09-21T10:24:26.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:25 smithi032 ceph-mon[105748]: 1.0 scrub ok 2024-09-21T10:24:26.244 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:26.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.270 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:26.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:26.277 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1133,"stamp":"2024-09-21T10:24:24.557626+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4521895,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":6209,"num_read_kb":13463,"num_write":1024,"num_write_kb":13942,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":296,"num_bytes_recovered":19467053,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2674,"ondisk_log_size":2674,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5279540,"kb_used_data":508600,"kb_used_omap":40,"kb_used_meta":4770775,"kb_avail":744681676,"statfs":{"total":767960285184,"available":762554036224,"internally_reserved":0,"allocated":520806400,"data_stored":505229055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41824,"internal_metadata":4885273760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-274010,"num_objects":-1,"num_object_clones":0,"num_object_copies":-3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":923709,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000811"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":631,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.815516+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:24:23.815516+0000","last_peered":"2024-09-21T10:24:23.815516+0000","last_clean":"2024-09-21T10:24:23.815516+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:24:23.815516+0000","last_undegraded":"2024-09-21T10:24:23.815516+0000","last_fullsized":"2024-09-21T10:24:23.815516+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":376,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.809491+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:24:16.809491+0000","last_peered":"2024-09-21T10:24:16.809491+0000","last_clean":"2024-09-21T10:24:16.809491+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:24:16.809491+0000","last_undegraded":"2024-09-21T10:24:16.809491+0000","last_fullsized":"2024-09-21T10:24:16.809491+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.806797+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:24:16.806797+0000","last_peered":"2024-09-21T10:24:16.806797+0000","last_clean":"2024-09-21T10:24:16.806797+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:24:16.806797+0000","last_undegraded":"2024-09-21T10:24:16.806797+0000","last_fullsized":"2024-09-21T10:24:16.806797+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":67,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.513945+0000","last_change":"2024-09-21T10:22:01.318638+0000","last_active":"2024-09-21T10:24:21.513945+0000","last_peered":"2024-09-21T10:24:21.513945+0000","last_clean":"2024-09-21T10:24:21.513945+0000","last_became_active":"2024-09-21T10:22:01.317087+0000","last_became_peered":"2024-09-21T10:22:01.317087+0000","last_unstale":"2024-09-21T10:24:21.513945+0000","last_undegraded":"2024-09-21T10:24:21.513945+0000","last_fullsized":"2024-09-21T10:24:21.513945+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:25.334726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.513534+0000","last_change":"2024-09-21T10:22:01.318978+0000","last_active":"2024-09-21T10:24:21.513534+0000","last_peered":"2024-09-21T10:24:21.513534+0000","last_clean":"2024-09-21T10:24:21.513534+0000","last_became_active":"2024-09-21T10:22:01.318255+0000","last_became_peered":"2024-09-21T10:22:01.318255+0000","last_unstale":"2024-09-21T10:24:21.513534+0000","last_undegraded":"2024-09-21T10:24:21.513534+0000","last_fullsized":"2024-09-21T10:24:21.513534+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:55.084088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":347,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808712+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:24:16.808712+0000","last_peered":"2024-09-21T10:24:16.808712+0000","last_clean":"2024-09-21T10:24:16.808712+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:24:16.808712+0000","last_undegraded":"2024-09-21T10:24:16.808712+0000","last_fullsized":"2024-09-21T10:24:16.808712+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":652,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.814870+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:24:23.814870+0000","last_peered":"2024-09-21T10:24:23.814870+0000","last_clean":"2024-09-21T10:24:23.814870+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:24:23.814870+0000","last_undegraded":"2024-09-21T10:24:23.814870+0000","last_fullsized":"2024-09-21T10:24:23.814870+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":428,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:17.221526+0000","last_change":"2024-09-21T10:22:01.331612+0000","last_active":"2024-09-21T10:24:17.221526+0000","last_peered":"2024-09-21T10:24:17.221526+0000","last_clean":"2024-09-21T10:24:17.221526+0000","last_became_active":"2024-09-21T10:22:01.325282+0000","last_became_peered":"2024-09-21T10:22:01.325282+0000","last_unstale":"2024-09-21T10:24:17.221526+0000","last_undegraded":"2024-09-21T10:24:17.221526+0000","last_fullsized":"2024-09-21T10:24:17.221526+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:23.383106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":63,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808936+0000","last_change":"2024-09-21T10:22:01.321467+0000","last_active":"2024-09-21T10:24:16.808936+0000","last_peered":"2024-09-21T10:24:16.808936+0000","last_clean":"2024-09-21T10:24:16.808936+0000","last_became_active":"2024-09-21T10:22:01.318270+0000","last_became_peered":"2024-09-21T10:22:01.318270+0000","last_unstale":"2024-09-21T10:24:16.808936+0000","last_undegraded":"2024-09-21T10:24:16.808936+0000","last_fullsized":"2024-09-21T10:24:16.808936+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:44.059553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":80,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.806528+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:24:16.806528+0000","last_peered":"2024-09-21T10:24:16.806528+0000","last_clean":"2024-09-21T10:24:16.806528+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:24:16.806528+0000","last_undegraded":"2024-09-21T10:24:16.806528+0000","last_fullsized":"2024-09-21T10:24:16.806528+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":50,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808984+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:24:16.808984+0000","last_peered":"2024-09-21T10:24:16.808984+0000","last_clean":"2024-09-21T10:24:16.808984+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:24:16.808984+0000","last_undegraded":"2024-09-21T10:24:16.808984+0000","last_fullsized":"2024-09-21T10:24:16.808984+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.459150+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:24:16.459150+0000","last_peered":"2024-09-21T10:24:16.459150+0000","last_clean":"2024-09-21T10:24:16.459150+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:24:16.459150+0000","last_undegraded":"2024-09-21T10:24:16.459150+0000","last_fullsized":"2024-09-21T10:24:16.459150+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":372,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.809140+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:24:16.809140+0000","last_peered":"2024-09-21T10:24:16.809140+0000","last_clean":"2024-09-21T10:24:16.809140+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:24:16.809140+0000","last_undegraded":"2024-09-21T10:24:16.809140+0000","last_fullsized":"2024-09-21T10:24:16.809140+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":391,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:17.221532+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:24:17.221532+0000","last_peered":"2024-09-21T10:24:17.221532+0000","last_clean":"2024-09-21T10:24:17.221532+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:24:17.221532+0000","last_undegraded":"2024-09-21T10:24:17.221532+0000","last_fullsized":"2024-09-21T10:24:17.221532+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":305,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.806641+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:24:16.806641+0000","last_peered":"2024-09-21T10:24:16.806641+0000","last_clean":"2024-09-21T10:24:16.806641+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:24:16.806641+0000","last_undegraded":"2024-09-21T10:24:16.806641+0000","last_fullsized":"2024-09-21T10:24:16.806641+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":237,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.513679+0000","last_change":"2024-09-21T10:19:50.585724+0000","last_active":"2024-09-21T10:24:21.513679+0000","last_peered":"2024-09-21T10:24:21.513679+0000","last_clean":"2024-09-21T10:24:21.513679+0000","last_became_active":"2024-09-21T10:19:50.585315+0000","last_became_peered":"2024-09-21T10:19:50.585315+0000","last_unstale":"2024-09-21T10:24:21.513679+0000","last_undegraded":"2024-09-21T10:24:21.513679+0000","last_fullsized":"2024-09-21T10:24:21.513679+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:26.618970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":640,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808526+0000","last_change":"2024-09-21T10:24:11.164077+0000","last_active":"2024-09-21T10:24:16.808526+0000","last_peered":"2024-09-21T10:24:16.808526+0000","last_clean":"2024-09-21T10:24:16.808526+0000","last_became_active":"2024-09-21T10:22:04.028853+0000","last_became_peered":"2024-09-21T10:22:04.028853+0000","last_unstale":"2024-09-21T10:24:16.808526+0000","last_undegraded":"2024-09-21T10:24:16.808526+0000","last_fullsized":"2024-09-21T10:24:16.808526+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:31.180490+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":682,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.814263+0000","last_change":"2024-09-21T10:22:01.335303+0000","last_active":"2024-09-21T10:24:23.814263+0000","last_peered":"2024-09-21T10:24:23.814263+0000","last_clean":"2024-09-21T10:24:23.814263+0000","last_became_active":"2024-09-21T10:22:01.325298+0000","last_became_peered":"2024-09-21T10:22:01.325298+0000","last_unstale":"2024-09-21T10:24:23.814263+0000","last_undegraded":"2024-09-21T10:24:23.814263+0000","last_fullsized":"2024-09-21T10:24:23.814263+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:01.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.281989+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:24:18.281989+0000","last_peered":"2024-09-21T10:24:18.281989+0000","last_clean":"2024-09-21T10:24:18.281989+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:24:18.281989+0000","last_undegraded":"2024-09-21T10:24:18.281989+0000","last_fullsized":"2024-09-21T10:24:18.281989+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":456,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808726+0000","last_change":"2024-09-21T10:23:47.785204+0000","last_active":"2024-09-21T10:24:16.808726+0000","last_peered":"2024-09-21T10:24:16.808726+0000","last_clean":"2024-09-21T10:24:16.808726+0000","last_became_active":"2024-09-21T10:23:47.784780+0000","last_became_peered":"2024-09-21T10:23:47.784780+0000","last_unstale":"2024-09-21T10:24:16.808726+0000","last_undegraded":"2024-09-21T10:24:16.808726+0000","last_fullsized":"2024-09-21T10:24:16.808726+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:19:15.365774+0000","last_clean_scrub_stamp":"2024-09-21T10:19:15.365774+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":381,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:17.221753+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:24:17.221753+0000","last_peered":"2024-09-21T10:24:17.221753+0000","last_clean":"2024-09-21T10:24:17.221753+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:24:17.221753+0000","last_undegraded":"2024-09-21T10:24:17.221753+0000","last_fullsized":"2024-09-21T10:24:17.221753+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":69,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.282323+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:24:18.282323+0000","last_peered":"2024-09-21T10:24:18.282323+0000","last_clean":"2024-09-21T10:24:18.282323+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:24:18.282323+0000","last_undegraded":"2024-09-21T10:24:18.282323+0000","last_fullsized":"2024-09-21T10:24:18.282323+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.459068+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:24:16.459068+0000","last_peered":"2024-09-21T10:24:16.459068+0000","last_clean":"2024-09-21T10:24:16.459068+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:24:16.459068+0000","last_undegraded":"2024-09-21T10:24:16.459068+0000","last_fullsized":"2024-09-21T10:24:16.459068+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.282414+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:24:18.282414+0000","last_peered":"2024-09-21T10:24:18.282414+0000","last_clean":"2024-09-21T10:24:18.282414+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:24:18.282414+0000","last_undegraded":"2024-09-21T10:24:18.282414+0000","last_fullsized":"2024-09-21T10:24:18.282414+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":629,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808799+0000","last_change":"2024-09-21T10:24:10.058618+0000","last_active":"2024-09-21T10:24:16.808799+0000","last_peered":"2024-09-21T10:24:16.808799+0000","last_clean":"2024-09-21T10:24:16.808799+0000","last_became_active":"2024-09-21T10:22:04.030774+0000","last_became_peered":"2024-09-21T10:22:04.030774+0000","last_unstale":"2024-09-21T10:24:16.808799+0000","last_undegraded":"2024-09-21T10:24:16.808799+0000","last_fullsized":"2024-09-21T10:24:16.808799+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:22.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:22.489745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":660,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.814071+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:24:23.814071+0000","last_peered":"2024-09-21T10:24:23.814071+0000","last_clean":"2024-09-21T10:24:23.814071+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:24:23.814071+0000","last_undegraded":"2024-09-21T10:24:23.814071+0000","last_fullsized":"2024-09-21T10:24:23.814071+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":344,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.349906+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:24:18.349906+0000","last_peered":"2024-09-21T10:24:18.349906+0000","last_clean":"2024-09-21T10:24:18.349906+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:24:18.349906+0000","last_undegraded":"2024-09-21T10:24:18.349906+0000","last_fullsized":"2024-09-21T10:24:18.349906+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":629,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808633+0000","last_change":"2024-09-21T10:24:10.058393+0000","last_active":"2024-09-21T10:24:16.808633+0000","last_peered":"2024-09-21T10:24:16.808633+0000","last_clean":"2024-09-21T10:24:16.808633+0000","last_became_active":"2024-09-21T10:22:04.028938+0000","last_became_peered":"2024-09-21T10:22:04.028938+0000","last_unstale":"2024-09-21T10:24:16.808633+0000","last_undegraded":"2024-09-21T10:24:16.808633+0000","last_fullsized":"2024-09-21T10:24:16.808633+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:12.388304+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:12.388304+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":685,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:17.811137+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:24:17.811137+0000","last_peered":"2024-09-21T10:24:17.811137+0000","last_clean":"2024-09-21T10:24:17.811137+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:24:17.811137+0000","last_undegraded":"2024-09-21T10:24:17.811137+0000","last_fullsized":"2024-09-21T10:24:17.811137+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":591,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.814012+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:24:23.814012+0000","last_peered":"2024-09-21T10:24:23.814012+0000","last_clean":"2024-09-21T10:24:23.814012+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:24:23.814012+0000","last_undegraded":"2024-09-21T10:24:23.814012+0000","last_fullsized":"2024-09-21T10:24:23.814012+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":348,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:17.221941+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:24:17.221941+0000","last_peered":"2024-09-21T10:24:17.221941+0000","last_clean":"2024-09-21T10:24:17.221941+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:24:17.221941+0000","last_undegraded":"2024-09-21T10:24:17.221941+0000","last_fullsized":"2024-09-21T10:24:17.221941+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":629,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808985+0000","last_change":"2024-09-21T10:24:10.058503+0000","last_active":"2024-09-21T10:24:16.808985+0000","last_peered":"2024-09-21T10:24:16.808985+0000","last_clean":"2024-09-21T10:24:16.808985+0000","last_became_active":"2024-09-21T10:22:04.031125+0000","last_became_peered":"2024-09-21T10:22:04.031125+0000","last_unstale":"2024-09-21T10:24:16.808985+0000","last_undegraded":"2024-09-21T10:24:16.808985+0000","last_fullsized":"2024-09-21T10:24:16.808985+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:19:19.450903+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1013,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808857+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:24:16.808857+0000","last_peered":"2024-09-21T10:24:16.808857+0000","last_clean":"2024-09-21T10:24:16.808857+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:24:16.808857+0000","last_undegraded":"2024-09-21T10:24:16.808857+0000","last_fullsized":"2024-09-21T10:24:16.808857+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":235,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.511080+0000","last_change":"2024-09-21T10:19:51.555124+0000","last_active":"2024-09-21T10:24:21.511080+0000","last_peered":"2024-09-21T10:24:21.511080+0000","last_clean":"2024-09-21T10:24:21.511080+0000","last_became_active":"2024-09-21T10:19:51.554766+0000","last_became_peered":"2024-09-21T10:19:51.554766+0000","last_unstale":"2024-09-21T10:24:21.511080+0000","last_undegraded":"2024-09-21T10:24:21.511080+0000","last_fullsized":"2024-09-21T10:24:21.511080+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:52.832950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.513312+0000","last_change":"2024-09-21T10:22:01.316140+0000","last_active":"2024-09-21T10:24:21.513312+0000","last_peered":"2024-09-21T10:24:21.513312+0000","last_clean":"2024-09-21T10:24:21.513312+0000","last_became_active":"2024-09-21T10:22:01.315408+0000","last_became_peered":"2024-09-21T10:22:01.315408+0000","last_unstale":"2024-09-21T10:24:21.513312+0000","last_undegraded":"2024-09-21T10:24:21.513312+0000","last_fullsized":"2024-09-21T10:24:21.513312+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:07.322279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":640,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808705+0000","last_change":"2024-09-21T10:24:15.203963+0000","last_active":"2024-09-21T10:24:16.808705+0000","last_peered":"2024-09-21T10:24:16.808705+0000","last_clean":"2024-09-21T10:24:16.808705+0000","last_became_active":"2024-09-21T10:22:04.030000+0000","last_became_peered":"2024-09-21T10:22:04.030000+0000","last_unstale":"2024-09-21T10:24:16.808705+0000","last_undegraded":"2024-09-21T10:24:16.808705+0000","last_fullsized":"2024-09-21T10:24:16.808705+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:24.512145+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1181,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.817411+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:24:23.817411+0000","last_peered":"2024-09-21T10:24:23.817411+0000","last_clean":"2024-09-21T10:24:23.817411+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:24:23.817411+0000","last_undegraded":"2024-09-21T10:24:23.817411+0000","last_fullsized":"2024-09-21T10:24:23.817411+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":331,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.349684+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:24:18.349684+0000","last_peered":"2024-09-21T10:24:18.349684+0000","last_clean":"2024-09-21T10:24:18.349684+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:24:18.349684+0000","last_undegraded":"2024-09-21T10:24:18.349684+0000","last_fullsized":"2024-09-21T10:24:18.349684+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":361,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.349942+0000","last_change":"2024-09-21T10:22:01.317933+0000","last_active":"2024-09-21T10:24:18.349942+0000","last_peered":"2024-09-21T10:24:18.349942+0000","last_clean":"2024-09-21T10:24:18.349942+0000","last_became_active":"2024-09-21T10:22:01.317536+0000","last_became_peered":"2024-09-21T10:22:01.317536+0000","last_unstale":"2024-09-21T10:24:18.349942+0000","last_undegraded":"2024-09-21T10:24:18.349942+0000","last_fullsized":"2024-09-21T10:24:18.349942+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:34.312724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":640,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808895+0000","last_change":"2024-09-21T10:24:14.203857+0000","last_active":"2024-09-21T10:24:16.808895+0000","last_peered":"2024-09-21T10:24:16.808895+0000","last_clean":"2024-09-21T10:24:16.808895+0000","last_became_active":"2024-09-21T10:22:04.029353+0000","last_became_peered":"2024-09-21T10:22:04.029353+0000","last_unstale":"2024-09-21T10:24:16.808895+0000","last_undegraded":"2024-09-21T10:24:16.808895+0000","last_fullsized":"2024-09-21T10:24:16.808895+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:14.203777+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:14.203777+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:02:35.196047+0000","scrub_duration":0.0013739399999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1116,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:17.808014+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:24:17.808014+0000","last_peered":"2024-09-21T10:24:17.808014+0000","last_clean":"2024-09-21T10:24:17.808014+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:24:17.808014+0000","last_undegraded":"2024-09-21T10:24:17.808014+0000","last_fullsized":"2024-09-21T10:24:17.808014+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.459840+0000","last_change":"2024-09-21T10:19:50.581529+0000","last_active":"2024-09-21T10:24:16.459840+0000","last_peered":"2024-09-21T10:24:16.459840+0000","last_clean":"2024-09-21T10:24:16.459840+0000","last_became_active":"2024-09-21T10:19:50.581119+0000","last_became_peered":"2024-09-21T10:19:50.581119+0000","last_unstale":"2024-09-21T10:24:16.459840+0000","last_undegraded":"2024-09-21T10:24:16.459840+0000","last_fullsized":"2024-09-21T10:24:16.459840+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:13.334483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":331,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.349724+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:24:18.349724+0000","last_peered":"2024-09-21T10:24:18.349724+0000","last_clean":"2024-09-21T10:24:18.349724+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:24:18.349724+0000","last_undegraded":"2024-09-21T10:24:18.349724+0000","last_fullsized":"2024-09-21T10:24:18.349724+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":629,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.809080+0000","last_change":"2024-09-21T10:24:10.059191+0000","last_active":"2024-09-21T10:24:16.809080+0000","last_peered":"2024-09-21T10:24:16.809080+0000","last_clean":"2024-09-21T10:24:16.809080+0000","last_became_active":"2024-09-21T10:22:04.029014+0000","last_became_peered":"2024-09-21T10:22:04.029014+0000","last_unstale":"2024-09-21T10:24:16.809080+0000","last_undegraded":"2024-09-21T10:24:16.809080+0000","last_fullsized":"2024-09-21T10:24:16.809080+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":320,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.806733+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:24:16.806733+0000","last_peered":"2024-09-21T10:24:16.806733+0000","last_clean":"2024-09-21T10:24:16.806733+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:24:16.806733+0000","last_undegraded":"2024-09-21T10:24:16.806733+0000","last_fullsized":"2024-09-21T10:24:16.806733+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":940,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.349917+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:24:18.349917+0000","last_peered":"2024-09-21T10:24:18.349917+0000","last_clean":"2024-09-21T10:24:18.349917+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:24:18.349917+0000","last_undegraded":"2024-09-21T10:24:18.349917+0000","last_fullsized":"2024-09-21T10:24:18.349917+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":367,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.349653+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:24:18.349653+0000","last_peered":"2024-09-21T10:24:18.349653+0000","last_clean":"2024-09-21T10:24:18.349653+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:24:18.349653+0000","last_undegraded":"2024-09-21T10:24:18.349653+0000","last_fullsized":"2024-09-21T10:24:18.349653+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":640,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.809065+0000","last_change":"2024-09-21T10:24:13.176097+0000","last_active":"2024-09-21T10:24:16.809065+0000","last_peered":"2024-09-21T10:24:16.809065+0000","last_clean":"2024-09-21T10:24:16.809065+0000","last_became_active":"2024-09-21T10:22:04.031343+0000","last_became_peered":"2024-09-21T10:22:04.031343+0000","last_unstale":"2024-09-21T10:24:16.809065+0000","last_undegraded":"2024-09-21T10:24:16.809065+0000","last_fullsized":"2024-09-21T10:24:16.809065+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:46.096244+0000","scrub_duration":0.0017209739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":349,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.349849+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:24:18.349849+0000","last_peered":"2024-09-21T10:24:18.349849+0000","last_clean":"2024-09-21T10:24:18.349849+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:24:18.349849+0000","last_undegraded":"2024-09-21T10:24:18.349849+0000","last_fullsized":"2024-09-21T10:24:18.349849+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":939,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.809183+0000","last_change":"2024-09-21T10:22:01.336961+0000","last_active":"2024-09-21T10:24:16.809183+0000","last_peered":"2024-09-21T10:24:16.809183+0000","last_clean":"2024-09-21T10:24:16.809183+0000","last_became_active":"2024-09-21T10:22:01.320079+0000","last_became_peered":"2024-09-21T10:22:01.320079+0000","last_unstale":"2024-09-21T10:24:16.809183+0000","last_undegraded":"2024-09-21T10:24:16.809183+0000","last_fullsized":"2024-09-21T10:24:16.809183+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:54.037388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":234,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.513659+0000","last_change":"2024-09-21T10:20:13.088557+0000","last_active":"2024-09-21T10:24:21.513659+0000","last_peered":"2024-09-21T10:24:21.513659+0000","last_clean":"2024-09-21T10:24:21.513659+0000","last_became_active":"2024-09-21T10:20:13.088123+0000","last_became_peered":"2024-09-21T10:20:13.088123+0000","last_unstale":"2024-09-21T10:24:21.513659+0000","last_undegraded":"2024-09-21T10:24:21.513659+0000","last_fullsized":"2024-09-21T10:24:21.513659+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:49.475499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":640,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:17.223323+0000","last_change":"2024-09-21T10:24:17.223323+0000","last_active":"2024-09-21T10:24:17.223323+0000","last_peered":"2024-09-21T10:24:17.223323+0000","last_clean":"2024-09-21T10:24:17.223323+0000","last_became_active":"2024-09-21T10:22:04.031881+0000","last_became_peered":"2024-09-21T10:22:04.031881+0000","last_unstale":"2024-09-21T10:24:17.223323+0000","last_undegraded":"2024-09-21T10:24:17.223323+0000","last_fullsized":"2024-09-21T10:24:17.223323+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:17.223224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:17.223224+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:43:27.226194+0000","scrub_duration":0.0030208050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":665,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.815054+0000","last_change":"2024-09-21T10:21:34.876196+0000","last_active":"2024-09-21T10:24:23.815054+0000","last_peered":"2024-09-21T10:24:23.815054+0000","last_clean":"2024-09-21T10:24:23.815054+0000","last_became_active":"2024-09-21T10:21:34.868031+0000","last_became_peered":"2024-09-21T10:21:34.868031+0000","last_unstale":"2024-09-21T10:24:23.815054+0000","last_undegraded":"2024-09-21T10:24:23.815054+0000","last_fullsized":"2024-09-21T10:24:23.815054+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:58.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1003,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.349432+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:24:18.349432+0000","last_peered":"2024-09-21T10:24:18.349432+0000","last_clean":"2024-09-21T10:24:18.349432+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:24:18.349432+0000","last_undegraded":"2024-09-21T10:24:18.349432+0000","last_fullsized":"2024-09-21T10:24:18.349432+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":661,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.813852+0000","last_change":"2024-09-21T10:22:01.329400+0000","last_active":"2024-09-21T10:24:23.813852+0000","last_peered":"2024-09-21T10:24:23.813852+0000","last_clean":"2024-09-21T10:24:23.813852+0000","last_became_active":"2024-09-21T10:22:01.328887+0000","last_became_peered":"2024-09-21T10:22:01.328887+0000","last_unstale":"2024-09-21T10:24:23.813852+0000","last_undegraded":"2024-09-21T10:24:23.813852+0000","last_fullsized":"2024-09-21T10:24:23.813852+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:09.626637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":628,"reported_epoch":377,"state":"peering","last_fresh":"2024-09-21T10:24:22.820860+0000","last_change":"2024-09-21T10:24:23.923693+0000","last_active":"2024-09-21T10:24:23.923693+0000","last_peered":"2024-09-21T10:24:22.820860+0000","last_clean":"2024-09-21T10:24:18.229289+0000","last_became_active":"2024-09-21T10:24:22.817580+0000","last_became_peered":"2024-09-21T10:24:22.817580+0000","last_unstale":"2024-09-21T10:24:22.820860+0000","last_undegraded":"2024-09-21T10:24:22.820860+0000","last_fullsized":"2024-09-21T10:24:22.812520+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:18.230314+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":923709,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":71,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.513430+0000","last_change":"2024-09-21T10:23:24.764361+0000","last_active":"2024-09-21T10:24:21.513430+0000","last_peered":"2024-09-21T10:24:21.513430+0000","last_clean":"2024-09-21T10:24:21.513430+0000","last_became_active":"2024-09-21T10:22:01.317312+0000","last_became_peered":"2024-09-21T10:22:01.317312+0000","last_unstale":"2024-09-21T10:24:21.513430+0000","last_undegraded":"2024-09-21T10:24:21.513430+0000","last_fullsized":"2024-09-21T10:24:21.513430+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:32:39.440581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":683,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:17.221960+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:24:17.221960+0000","last_peered":"2024-09-21T10:24:17.221960+0000","last_clean":"2024-09-21T10:24:17.221960+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:24:17.221960+0000","last_undegraded":"2024-09-21T10:24:17.221960+0000","last_fullsized":"2024-09-21T10:24:17.221960+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":345,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.459248+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:24:16.459248+0000","last_peered":"2024-09-21T10:24:16.459248+0000","last_clean":"2024-09-21T10:24:16.459248+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:24:16.459248+0000","last_undegraded":"2024-09-21T10:24:16.459248+0000","last_fullsized":"2024-09-21T10:24:16.459248+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":369,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.459902+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:24:16.459902+0000","last_peered":"2024-09-21T10:24:16.459902+0000","last_clean":"2024-09-21T10:24:16.459902+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:24:16.459902+0000","last_undegraded":"2024-09-21T10:24:16.459902+0000","last_fullsized":"2024-09-21T10:24:16.459902+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":78,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.461200+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:24:16.461200+0000","last_peered":"2024-09-21T10:24:16.461200+0000","last_clean":"2024-09-21T10:24:16.461200+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:24:16.461200+0000","last_undegraded":"2024-09-21T10:24:16.461200+0000","last_fullsized":"2024-09-21T10:24:16.461200+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.806534+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:24:16.806534+0000","last_peered":"2024-09-21T10:24:16.806534+0000","last_clean":"2024-09-21T10:24:16.806534+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:24:16.806534+0000","last_undegraded":"2024-09-21T10:24:16.806534+0000","last_fullsized":"2024-09-21T10:24:16.806534+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":52,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.808825+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:24:16.808825+0000","last_peered":"2024-09-21T10:24:16.808825+0000","last_clean":"2024-09-21T10:24:16.808825+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:24:16.808825+0000","last_undegraded":"2024-09-21T10:24:16.808825+0000","last_fullsized":"2024-09-21T10:24:16.808825+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":80,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.806625+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:24:16.806625+0000","last_peered":"2024-09-21T10:24:16.806625+0000","last_clean":"2024-09-21T10:24:16.806625+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:24:16.806625+0000","last_undegraded":"2024-09-21T10:24:16.806625+0000","last_fullsized":"2024-09-21T10:24:16.806625+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":584,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.813800+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:24:23.813800+0000","last_peered":"2024-09-21T10:24:23.813800+0000","last_clean":"2024-09-21T10:24:23.813800+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:24:23.813800+0000","last_undegraded":"2024-09-21T10:24:23.813800+0000","last_fullsized":"2024-09-21T10:24:23.813800+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":234,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.514123+0000","last_change":"2024-09-21T10:20:13.089473+0000","last_active":"2024-09-21T10:24:21.514123+0000","last_peered":"2024-09-21T10:24:21.514123+0000","last_clean":"2024-09-21T10:24:21.514123+0000","last_became_active":"2024-09-21T10:20:13.089111+0000","last_became_peered":"2024-09-21T10:20:13.089111+0000","last_unstale":"2024-09-21T10:24:21.514123+0000","last_undegraded":"2024-09-21T10:24:21.514123+0000","last_fullsized":"2024-09-21T10:24:21.514123+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:06.603284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":644,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.813738+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:24:23.813738+0000","last_peered":"2024-09-21T10:24:23.813738+0000","last_clean":"2024-09-21T10:24:23.813738+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:24:23.813738+0000","last_undegraded":"2024-09-21T10:24:23.813738+0000","last_fullsized":"2024-09-21T10:24:23.813738+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":387,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:17.221812+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:24:17.221812+0000","last_peered":"2024-09-21T10:24:17.221812+0000","last_clean":"2024-09-21T10:24:17.221812+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:24:17.221812+0000","last_undegraded":"2024-09-21T10:24:17.221812+0000","last_fullsized":"2024-09-21T10:24:17.221812+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":661,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.813949+0000","last_change":"2024-09-21T10:22:01.329690+0000","last_active":"2024-09-21T10:24:23.813949+0000","last_peered":"2024-09-21T10:24:23.813949+0000","last_clean":"2024-09-21T10:24:23.813949+0000","last_became_active":"2024-09-21T10:22:01.329033+0000","last_became_peered":"2024-09-21T10:22:01.329033+0000","last_unstale":"2024-09-21T10:24:23.813949+0000","last_undegraded":"2024-09-21T10:24:23.813949+0000","last_fullsized":"2024-09-21T10:24:23.813949+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:24.559524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":237,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.513635+0000","last_change":"2024-09-21T10:21:34.870704+0000","last_active":"2024-09-21T10:24:21.513635+0000","last_peered":"2024-09-21T10:24:21.513635+0000","last_clean":"2024-09-21T10:24:21.513635+0000","last_became_active":"2024-09-21T10:21:34.869835+0000","last_became_peered":"2024-09-21T10:21:34.869835+0000","last_unstale":"2024-09-21T10:24:21.513635+0000","last_undegraded":"2024-09-21T10:24:21.513635+0000","last_fullsized":"2024-09-21T10:24:21.513635+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:34.959915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":351,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-21T10:24:18.349556+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:24:18.349556+0000","last_peered":"2024-09-21T10:24:18.349556+0000","last_clean":"2024-09-21T10:24:18.349556+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:24:18.349556+0000","last_undegraded":"2024-09-21T10:24:18.349556+0000","last_fullsized":"2024-09-21T10:24:18.349556+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":651,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.814189+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:24:23.814189+0000","last_peered":"2024-09-21T10:24:23.814189+0000","last_clean":"2024-09-21T10:24:23.814189+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:24:23.814189+0000","last_undegraded":"2024-09-21T10:24:23.814189+0000","last_fullsized":"2024-09-21T10:24:23.814189+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":376,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.460565+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:24:16.460565+0000","last_peered":"2024-09-21T10:24:16.460565+0000","last_clean":"2024-09-21T10:24:16.460565+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:24:16.460565+0000","last_undegraded":"2024-09-21T10:24:16.460565+0000","last_fullsized":"2024-09-21T10:24:16.460565+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":650,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.816060+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:24:23.816060+0000","last_peered":"2024-09-21T10:24:23.816060+0000","last_clean":"2024-09-21T10:24:23.816060+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:24:23.816060+0000","last_undegraded":"2024-09-21T10:24:23.816060+0000","last_fullsized":"2024-09-21T10:24:23.816060+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":66,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.510158+0000","last_change":"2024-09-21T10:22:01.318549+0000","last_active":"2024-09-21T10:24:21.510158+0000","last_peered":"2024-09-21T10:24:21.510158+0000","last_clean":"2024-09-21T10:24:21.510158+0000","last_became_active":"2024-09-21T10:22:01.318071+0000","last_became_peered":"2024-09-21T10:22:01.318071+0000","last_unstale":"2024-09-21T10:24:21.510158+0000","last_undegraded":"2024-09-21T10:24:21.510158+0000","last_fullsized":"2024-09-21T10:24:21.510158+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:49.061492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":68,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.459728+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:24:16.459728+0000","last_peered":"2024-09-21T10:24:16.459728+0000","last_clean":"2024-09-21T10:24:16.459728+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:24:16.459728+0000","last_undegraded":"2024-09-21T10:24:16.459728+0000","last_fullsized":"2024-09-21T10:24:16.459728+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":312,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.806503+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:24:16.806503+0000","last_peered":"2024-09-21T10:24:16.806503+0000","last_clean":"2024-09-21T10:24:16.806503+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:24:16.806503+0000","last_undegraded":"2024-09-21T10:24:16.806503+0000","last_fullsized":"2024-09-21T10:24:16.806503+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":68,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.459199+0000","last_change":"2024-09-21T10:19:50.588363+0000","last_active":"2024-09-21T10:24:16.459199+0000","last_peered":"2024-09-21T10:24:16.459199+0000","last_clean":"2024-09-21T10:24:16.459199+0000","last_became_active":"2024-09-21T10:19:50.588107+0000","last_became_peered":"2024-09-21T10:19:50.588107+0000","last_unstale":"2024-09-21T10:24:16.459199+0000","last_undegraded":"2024-09-21T10:24:16.459199+0000","last_fullsized":"2024-09-21T10:24:16.459199+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:05.698018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":290,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.806817+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:24:16.806817+0000","last_peered":"2024-09-21T10:24:16.806817+0000","last_clean":"2024-09-21T10:24:16.806817+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:24:16.806817+0000","last_undegraded":"2024-09-21T10:24:16.806817+0000","last_fullsized":"2024-09-21T10:24:16.806817+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":664,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.814425+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:24:23.814425+0000","last_peered":"2024-09-21T10:24:23.814425+0000","last_clean":"2024-09-21T10:24:23.814425+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:24:23.814425+0000","last_undegraded":"2024-09-21T10:24:23.814425+0000","last_fullsized":"2024-09-21T10:24:23.814425+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":626,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-09-21T10:24:23.815297+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:24:23.815297+0000","last_peered":"2024-09-21T10:24:23.815297+0000","last_clean":"2024-09-21T10:24:23.815297+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:24:23.815297+0000","last_undegraded":"2024-09-21T10:24:23.815297+0000","last_fullsized":"2024-09-21T10:24:23.815297+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-09-21T10:24:16.459018+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:24:16.459018+0000","last_peered":"2024-09-21T10:24:16.459018+0000","last_clean":"2024-09-21T10:24:16.459018+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:24:16.459018+0000","last_undegraded":"2024-09-21T10:24:16.459018+0000","last_fullsized":"2024-09-21T10:24:16.459018+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":73,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-09-21T10:24:21.513949+0000","last_change":"2024-09-21T10:23:24.753979+0000","last_active":"2024-09-21T10:24:21.513949+0000","last_peered":"2024-09-21T10:24:21.513949+0000","last_clean":"2024-09-21T10:24:21.513949+0000","last_became_active":"2024-09-21T10:22:01.316569+0000","last_became_peered":"2024-09-21T10:22:01.316569+0000","last_unstale":"2024-09-21T10:24:21.513949+0000","last_undegraded":"2024-09-21T10:24:21.513949+0000","last_fullsized":"2024-09-21T10:24:21.513949+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:35:44.904752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":6328,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24225,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":322018,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":212,"num_read_kb":453,"num_write":116,"num_write_kb":1317,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2683490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3235840,"data_stored":3214528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":6}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234416,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":618048,"kb_used_data":9388,"kb_used_omap":1,"kb_used_meta":608638,"kb_avail":93127104,"statfs":{"total":95995035648,"available":95362154496,"internally_reserved":0,"allocated":9613312,"data_stored":7657684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":623245771},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496054,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1092400,"kb_used_data":461608,"kb_used_omap":1,"kb_used_meta":630782,"kb_avail":92652752,"statfs":{"total":95995035648,"available":94876418048,"internally_reserved":0,"allocated":472686592,"data_stored":470735264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":645921223},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403954,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307376,"kb_used_data":4700,"kb_used_omap":7,"kb_used_meta":302648,"kb_avail":93437776,"statfs":{"total":95995035648,"available":95680282624,"internally_reserved":0,"allocated":4812800,"data_stored":2869042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":309912472},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019327,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":613368,"kb_used_data":4712,"kb_used_omap":1,"kb_used_meta":608638,"kb_avail":93131784,"statfs":{"total":95995035648,"available":95366946816,"internally_reserved":0,"allocated":4825088,"data_stored":2873273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":623245766},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435512,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":798916,"kb_used_data":5048,"kb_used_omap":5,"kb_used_meta":793850,"kb_avail":92946236,"statfs":{"total":95995035648,"available":95176945664,"internally_reserved":0,"allocated":5169152,"data_stored":3221237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":812902440},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599039,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":712784,"kb_used_data":4988,"kb_used_omap":7,"kb_used_meta":707768,"kb_avail":93032368,"statfs":{"total":95995035648,"available":95265144832,"internally_reserved":0,"allocated":5107712,"data_stored":3156463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":724754893},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762565,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":640224,"kb_used_data":8784,"kb_used_omap":7,"kb_used_meta":631416,"kb_avail":93104928,"statfs":{"total":95995035648,"available":95339446272,"internally_reserved":0,"allocated":8994816,"data_stored":7062433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":646570237},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926092,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":496424,"kb_used_data":9372,"kb_used_omap":7,"kb_used_meta":487032,"kb_avail":93248728,"statfs":{"total":95995035648,"available":95486697472,"internally_reserved":0,"allocated":9596928,"data_stored":7653659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":498720958},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:24:26.370 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:26.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.394 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:26.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:26.493 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:24:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:24:26.495 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:26.514 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:24:26.514 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:24:26.518 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:26.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:26.532 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:24:26.532 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:24:26.550 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:24:26.550 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:24:26.556 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:26.571 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:24:26.572 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:24:26.598 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:24:26.598 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:24:26.599 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:26.615 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:24:26.615 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:24:26.619 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:26.632 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:24:26.632 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:24:26.642 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:26.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:26.744 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:26.766 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:26.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:26.856 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:24:26.868 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:26.891 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:26.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:27.015 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:27.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:27.061 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:27.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:26 smithi032 ceph-mon[101299]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T10:24:27.097 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1096187405' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:27.097 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:26 smithi032 ceph-mon[105748]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T10:24:27.097 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1096187405' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:27.117 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:27.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.140 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:27.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:27.169 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:26 smithi045 ceph-mon[95976]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T10:24:27.169 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1096187405' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:27.241 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:27.263 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:27.364 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.387 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:27.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:27.488 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:27.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:27.613 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:27.636 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:27.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:27.737 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:27.760 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:27.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:27.862 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:27.884 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:27.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[101299]: 1.5 scrub starts 2024-09-21T10:24:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[101299]: 1.5 scrub ok 2024-09-21T10:24:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[101299]: pgmap v1136: 99 pgs: 9 creating+peering, 7 unknown, 1 peering, 82 active+clean; 4.4 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 38 KiB/s rd, 110 KiB/s wr, 28 op/s; 291 KiB/s, 0 objects/s recovering 2024-09-21T10:24:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[101299]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:24:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[101299]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T10:24:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[101299]: 1.4 scrub starts 2024-09-21T10:24:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[101299]: 1.4 scrub ok 2024-09-21T10:24:27.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[105748]: 1.5 scrub starts 2024-09-21T10:24:27.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[105748]: 1.5 scrub ok 2024-09-21T10:24:27.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[105748]: pgmap v1136: 99 pgs: 9 creating+peering, 7 unknown, 1 peering, 82 active+clean; 4.4 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 38 KiB/s rd, 110 KiB/s wr, 28 op/s; 291 KiB/s, 0 objects/s recovering 2024-09-21T10:24:27.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[105748]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:24:27.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[105748]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T10:24:27.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[105748]: 1.4 scrub starts 2024-09-21T10:24:27.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:27 smithi032 ceph-mon[105748]: 1.4 scrub ok 2024-09-21T10:24:27.986 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:28.009 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:28.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:28.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:28.132 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:28.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:28.203 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:27 smithi045 ceph-mon[95976]: 1.5 scrub starts 2024-09-21T10:24:28.203 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:27 smithi045 ceph-mon[95976]: 1.5 scrub ok 2024-09-21T10:24:28.203 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:27 smithi045 ceph-mon[95976]: pgmap v1136: 99 pgs: 9 creating+peering, 7 unknown, 1 peering, 82 active+clean; 4.4 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 38 KiB/s rd, 110 KiB/s wr, 28 op/s; 291 KiB/s, 0 objects/s recovering 2024-09-21T10:24:28.203 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:27 smithi045 ceph-mon[95976]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:24:28.203 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:27 smithi045 ceph-mon[95976]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T10:24:28.203 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:27 smithi045 ceph-mon[95976]: 1.4 scrub starts 2024-09-21T10:24:28.203 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:27 smithi045 ceph-mon[95976]: 1.4 scrub ok 2024-09-21T10:24:28.234 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:28.262 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:28.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:28.363 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:28.389 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:28.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:28.491 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:28.518 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:28.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:28.620 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:28.644 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:28.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:126173 8.d 8:bbe71d92:::benchmark_data_smithi032_233595_object26328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:21:52.546485+0000", 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 156.14350472199999, 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.416189632, 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.694 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 126173 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546485+0000", 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546485+0000", 2024-09-21T10:24:28.695 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546483+0000", 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546489+0000", 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4659999999999996e-06 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.696 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546501+0000", 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1599e-05 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546508+0000", 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1539999999999996e-06 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.599690+0000", 2024-09-21T10:24:28.698 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.053182116000000001 2024-09-21T10:24:28.698 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:24:28.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.601485+0000", 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0017949839999999999 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.233600+0000", 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.63211453500000003 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:24:28.699 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.236756+0000", 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031565199999999999 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.933863+0000", 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.69710698299999996 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.936865+0000", 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0030012799999999998 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:24:28.700 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.942213+0000", 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0053482879999999997 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.953349+0000", 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011136102 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.953630+0000", 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000281 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.954544+0000", 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00091381800000000005 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.955453+0000", 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00090889200000000001 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.956882+0000", 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0014294450000000001 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.962674+0000", 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0057920460000000003 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:127125 8.d 8:b41a31bb:::benchmark_data_smithi032_233595_object27280:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:21:56.132642+0000", 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 152.55734685499999, 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3750970520000001, 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.703 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 127125 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132642+0000", 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.704 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132642+0000", 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132640+0000", 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132649+0000", 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2989999999999997e-06 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132665+0000", 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6138999999999999e-05 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132673+0000", 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9570000000000002e-06 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.190478+0000", 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.057805095000000001 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.191875+0000", 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001396867 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.759185+0000", 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.56731008699999996 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.762972+0000", 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0037875370000000001 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.466882+0000", 2024-09-21T10:24:28.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.70391016799999995 2024-09-21T10:24:28.708 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.469411+0000", 2024-09-21T10:24:28.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0025282450000000001 2024-09-21T10:24:28.709 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.709 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.710 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:24:28.710 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.474173+0000", 2024-09-21T10:24:28.710 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0047627809999999998 2024-09-21T10:24:28.710 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.710 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.710 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.498463+0000", 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.024289439999999999 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.499344+0000", 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00088115700000000003 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.500162+0000", 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00081813700000000003 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.711 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.500964+0000", 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00080155899999999997 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.502290+0000", 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001326144 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.507739+0000", 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0054495749999999999 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "MOSDPGPush(8.d 348/339 [PushOp(8:bb1880f6:::benchmark_data_smithi032_233595_object29338:head, version: 314'1888, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:bb1880f6:::benchmark_data_smithi032_233595_object29338:head@314'1888, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:02.376911+0000", 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 146.31307788500001, 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5228334210000001, 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376911+0000", 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.713 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376911+0000", 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376911+0000", 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376950+0000", 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8979000000000001e-05 2024-09-21T10:24:28.714 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376964+0000", 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4409e-05 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376977+0000", 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.312e-05 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.889091+0000", 2024-09-21T10:24:28.715 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.51211345 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.889396+0000", 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00030527700000000001 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.899745+0000", 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010348697 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.716 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:129183 8.d 8:bb1880f6:::benchmark_data_smithi032_233595_object29338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:03.381897+0000", 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 145.30809221199999, 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2215354430000001, 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 129183 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.717 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381897+0000", 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381897+0000", 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381895+0000", 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381903+0000", 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.165e-06 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381917+0000", 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4945e-05 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.719 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381925+0000", 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5510000000000002e-06 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.598817+0000", 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.21689214900000001 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.600951+0000", 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002134141 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.720 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.900025+0000", 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.29907400099999998 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.902880+0000", 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0028542099999999998 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.571519+0000", 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.66863905599999995 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.721 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.573803+0000", 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022840120000000002 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.578058+0000", 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0042553729999999998 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.593724+0000", 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015666253000000002 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.722 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.593986+0000", 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00026158399999999998 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.594907+0000", 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00092106000000000002 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.595766+0000", 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00085888999999999996 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.723 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.597240+0000", 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0014741159999999999 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.603432+0000", 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0061925740000000002 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:129369 8.d 8:b32a093f:::benchmark_data_smithi032_233595_object29524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:04.040016+0000", 2024-09-21T10:24:28.724 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 144.64997296300001, 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.187289474, 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 129369 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040016+0000", 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.725 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040016+0000", 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040015+0000", 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040022+0000", 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7050000000000002e-06 2024-09-21T10:24:28.726 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040034+0000", 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2129e-05 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040042+0000", 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0779999999999996e-06 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.072059+0000", 2024-09-21T10:24:28.727 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.032017589999999999 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.073424+0000", 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001364765 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.524208+0000", 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.45078337699999999 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.527081+0000", 2024-09-21T10:24:28.728 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0028734640000000001 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.181266+0000", 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.65418491099999998 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.184177+0000", 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029113770000000001 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.189517+0000", 2024-09-21T10:24:28.729 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0053394569999999997 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.216225+0000", 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.026708104999999999 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.217266+0000", 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001040736 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.730 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.218049+0000", 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00078369599999999996 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.219162+0000", 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0011127120000000001 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.220543+0000", 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013809250000000001 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.731 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.227306+0000", 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0067628059999999997 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:130475 8.d 8:b0e43261:::benchmark_data_smithi032_233595_object30630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:07.833409+0000", 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 140.85658041100001, 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2543778620000001, 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.732 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 130475 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833409+0000", 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833409+0000", 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.733 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833407+0000", 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833414+0000", 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3419999999999996e-06 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833426+0000", 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1956999999999999e-05 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.734 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833431+0000", 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6250000000000004e-06 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.844081+0000", 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010650049999999999 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.845500+0000", 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001418348 2024-09-21T10:24:28.735 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:08.478678+0000", 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.63317818199999998 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:08.482182+0000", 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0035043309999999999 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.736 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.038948+0000", 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.55676618899999997 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.041858+0000", 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029101019999999999 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.049048+0000", 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0071896709999999999 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.737 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.077242+0000", 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.028194252 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.078305+0000", 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0010624269999999999 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.079245+0000", 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00093980400000000003 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.738 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.080257+0000", 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0010122460000000001 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.081835+0000", 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001577651 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.087787+0000", 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0059521790000000002 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.739 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:137893 8.d 8:bc2e2b1d:::benchmark_data_smithi032_233595_object38048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:28.962493+0000", 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 119.727495874, 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1154502959999999, 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 137893 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962493+0000", 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962493+0000", 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962492+0000", 2024-09-21T10:24:28.741 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962498+0000", 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.891e-06 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962509+0000", 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1324e-05 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.742 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962515+0000", 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9710000000000001e-06 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.986383+0000", 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.023867616000000001 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.987610+0000", 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001227204 2024-09-21T10:24:28.743 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:29.484804+0000", 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.49719403600000001 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:29.487315+0000", 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002511685 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.028673+0000", 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54135758599999995 2024-09-21T10:24:28.744 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.031518+0000", 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0028447749999999999 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.036720+0000", 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.005202351 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.064312+0000", 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.027591844000000001 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.745 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.065138+0000", 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00082640299999999997 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.065916+0000", 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00077757800000000001 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.068661+0000", 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002744771 2024-09-21T10:24:28.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.069887+0000", 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0012257920000000001 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.077944+0000", 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0080571140000000006 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.747 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:138528 8.d 8:b42d3168:::benchmark_data_smithi032_233595_object38683:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:30.651318+0000", 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 118.038671253, 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.17798106, 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 138528 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.748 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651318+0000", 2024-09-21T10:24:28.749 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.749 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.749 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.749 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651318+0000", 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651316+0000", 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651323+0000", 2024-09-21T10:24:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9249999999999998e-06 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651334+0000", 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0733999999999999e-05 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651342+0000", 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1759999999999993e-06 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.752 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.693935+0000", 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.042592334000000003 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.695200+0000", 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001265555 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.226631+0000", 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.53143101500000001 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.753 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.229128+0000", 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002496322 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.807497+0000", 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57836990700000002 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.809660+0000", 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0021629309999999999 2024-09-21T10:24:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.813036+0000", 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0033755149999999999 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.821737+0000", 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087014469999999993 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.822419+0000", 2024-09-21T10:24:28.755 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00068219000000000005 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.822626+0000", 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00020688099999999999 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.823351+0000", 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00072413800000000004 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.756 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.824503+0000", 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001152246 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.829299+0000", 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.004796285 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:139301 8.d 8:bd7927c9:::benchmark_data_smithi032_233595_object39456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-21T10:24:28.757 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:32.637011+0000", 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 116.052977992, 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99859100099999998, 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 139301 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637011+0000", 2024-09-21T10:24:28.758 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637011+0000", 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637010+0000", 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637016+0000", 2024-09-21T10:24:28.759 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6289999999999998e-06 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637026+0000", 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.094e-05 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637031+0000", 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8729999999999998e-06 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.760 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.652132+0000", 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015100891 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.653357+0000", 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001224356 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.105349+0000", 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.45199279599999997 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:24:28.761 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.107505+0000", 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0021551140000000001 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.605380+0000", 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.49787499699999999 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.607090+0000", 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001710575 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.762 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.610073+0000", 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029829330000000001 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.629265+0000", 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.019192331999999999 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.629458+0000", 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000192497 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.763 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.630105+0000", 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000647633 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.630687+0000", 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00058175499999999999 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.631651+0000", 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00096404699999999999 2024-09-21T10:24:28.764 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.635602+0000", 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039509749999999998 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174180 8.8 8:1cb2aae1:::benchmark_data_smithi032_233595_object74335:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666281+0000", 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 62.023708702, 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0395073349999999, 2024-09-21T10:24:28.765 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174180 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666281+0000", 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.766 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666281+0000", 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666279+0000", 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666288+0000", 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4409999999999993e-06 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.767 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666297+0000", 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.9350000000000006e-06 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666304+0000", 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3589999999999996e-06 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676089+0000", 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0097852890000000008 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.768 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676099+0000", 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4630000000000007e-06 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.705704+0000", 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.029605076 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.705788+0000", 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4192999999999998e-05 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.769 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174191 8.8 8:1067ce08:::benchmark_data_smithi032_233595_object74346:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666311+0000", 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 62.023677743999997, 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.039912868, 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174191 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.770 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666311+0000", 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666311+0000", 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666311+0000", 2024-09-21T10:24:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666313+0000", 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2850000000000001e-06 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666319+0000", 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3929999999999999e-06 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666323+0000", 2024-09-21T10:24:28.772 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4279999999999998e-06 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676281+0000", 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0099576779999999993 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676291+0000", 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6099999999999995e-06 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706038+0000", 2024-09-21T10:24:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0297476919999999 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706224+0000", 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000186152 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174200 8.8 8:1e9d63d6:::benchmark_data_smithi032_233595_object74355:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666327+0000", 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 62.023661998999998, 2024-09-21T10:24:28.774 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0398621889999999, 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174200 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666327+0000", 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.775 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666327+0000", 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666327+0000", 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666329+0000", 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9589999999999998e-06 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666334+0000", 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1170000000000003e-06 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666338+0000", 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.391e-06 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676317+0000", 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0099783319999999995 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.777 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676323+0000", 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7009999999999999e-06 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706125+0000", 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.029801084 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706189+0000", 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4919000000000006e-05 2024-09-21T10:24:28.778 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174202 8.8 8:1d58c5d6:::benchmark_data_smithi032_233595_object74357:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666343+0000", 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 62.023646722000002, 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.039984061, 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174202 2024-09-21T10:24:28.779 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666343+0000", 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666343+0000", 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666342+0000", 2024-09-21T10:24:28.780 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666344+0000", 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8360000000000001e-06 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666349+0000", 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9470000000000002e-06 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.781 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666353+0000", 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0729999999999998e-06 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676345+0000", 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0099917489999999994 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676351+0000", 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5019999999999998e-06 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.782 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706272+0000", 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.029920929 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706327+0000", 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4351999999999997e-05 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174205 8.8 8:1099f92f:::benchmark_data_smithi032_233595_object74360:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:24:28.783 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666357+0000", 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 62.023632567, 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0550764530000001, 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174205 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666357+0000", 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.784 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666357+0000", 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666356+0000", 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666358+0000", 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8560000000000001e-06 2024-09-21T10:24:28.785 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666363+0000", 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.096e-06 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666367+0000", 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0539999999999996e-06 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.786 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676801+0000", 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.01043375 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676830+0000", 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8911000000000002e-05 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706354+0000", 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.029523974 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.787 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706455+0000", 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010135499999999999 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721358+0000", 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014903035 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721433+0000", 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4714000000000003e-05 2024-09-21T10:24:28.788 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174244 8.8 8:1bf2b167:::benchmark_data_smithi032_233595_object74399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.722448+0000", 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 61.967540927999998, 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99908222599999996, 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.789 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174244 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722448+0000", 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722448+0000", 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.790 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722447+0000", 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722453+0000", 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6800000000000004e-06 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722469+0000", 2024-09-21T10:24:28.791 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5333e-05 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722476+0000", 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9970000000000001e-06 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.742937+0000", 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.020461098 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.792 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.742946+0000", 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2920000000000001e-06 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706508+0000", 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96356169400000002 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706631+0000", 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012306 2024-09-21T10:24:28.793 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706824+0000", 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000193325 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.717553+0000", 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010728946 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.717566+0000", 2024-09-21T10:24:28.794 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2446000000000001e-05 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721454+0000", 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0038887230000000002 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721483+0000", 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8660999999999999e-05 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.795 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721531+0000", 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7509000000000002e-05 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174257 8.8 8:1e3e855f:::benchmark_data_smithi032_233595_object74412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.737607+0000", 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 61.952382626000002, 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99780132499999996, 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.796 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174257 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737607+0000", 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.797 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737607+0000", 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737605+0000", 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737617+0000", 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2109e-05 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.798 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737629+0000", 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2557e-05 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737636+0000", 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7490000000000001e-06 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.747542+0000", 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0099056779999999994 2024-09-21T10:24:28.799 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.747552+0000", 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0183000000000001e-05 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.713711+0000", 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96615876499999997 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.713848+0000", 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013773400000000001 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.800 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.714001+0000", 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000152538 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721431+0000", 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0074296529999999996 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.728270+0000", 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0068392119999999999 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.801 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.735248+0000", 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0069787089999999996 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.735306+0000", 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7936000000000002e-05 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.735408+0000", 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000101568 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.802 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:191427 8.d 8:b66cbf8c:::benchmark_data_smithi032_233595_object91582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:47.766812+0000", 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 40.923177578999997, 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.050450109, 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 191427 2024-09-21T10:24:28.803 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766812+0000", 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766812+0000", 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.804 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766810+0000", 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766818+0000", 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5530000000000004e-06 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766834+0000", 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5931e-05 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.805 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766841+0000", 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0180000000000004e-06 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.774617+0000", 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0077766019999999996 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.774704+0000", 2024-09-21T10:24:28.806 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6446999999999994e-05 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.805821+0000", 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.031117743 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.805945+0000", 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012376499999999999 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.806069+0000", 2024-09-21T10:24:28.807 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012341899999999999 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.813666+0000", 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0075971850000000002 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817135+0000", 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0034688340000000001 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.808 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817206+0000", 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1890000000000005e-05 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817238+0000", 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1825000000000001e-05 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817262+0000", 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3470999999999999e-05 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.809 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:191444 8.d 8:b32f16b0:::benchmark_data_smithi032_233595_object91599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:47.787356+0000", 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 40.902633543, 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0395182649999999, 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 191444 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.810 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787356+0000", 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787356+0000", 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787354+0000", 2024-09-21T10:24:28.811 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787361+0000", 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7190000000000004e-06 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787379+0000", 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8063e-05 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787388+0000", 2024-09-21T10:24:28.812 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5990000000000002e-06 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787442+0000", 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4570000000000001e-05 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787452+0000", 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8090000000000004e-06 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.810170+0000", 2024-09-21T10:24:28.813 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0227181999999999 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.810272+0000", 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000102013 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.810342+0000", 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0140000000000003e-05 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817302+0000", 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0069597390000000004 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817312+0000", 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4050000000000006e-06 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826810+0000", 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094981749999999993 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.815 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826840+0000", 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0633000000000003e-05 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826874+0000", 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3541000000000003e-05 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:191477 8.d 8:b5a49fb0:::benchmark_data_smithi032_233595_object91632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:47.820759+0000", 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 40.869230090000002, 2024-09-21T10:24:28.816 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.006215826, 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 191477 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820759+0000", 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.817 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820759+0000", 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820757+0000", 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820765+0000", 2024-09-21T10:24:28.818 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6109999999999996e-06 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820777+0000", 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2283000000000001e-05 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820785+0000", 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1690000000000003e-06 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.819 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820832+0000", 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7435000000000002e-05 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820841+0000", 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.1039999999999996e-06 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.813711+0000", 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99286991099999999 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.820 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.813806+0000", 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4640000000000002e-05 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.813884+0000", 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8230000000000001e-05 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817319+0000", 2024-09-21T10:24:28.821 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0034351300000000002 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817350+0000", 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0809999999999998e-05 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826919+0000", 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0095689750000000004 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.822 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826950+0000", 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1075999999999999e-05 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826975+0000", 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5123999999999999e-05 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:191490 8.d 8:b4e2352a:::benchmark_data_smithi032_233595_object91645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-09-21T10:24:28.823 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:47.837183+0000", 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 40.852805992999997, 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98991871600000003, 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 191490 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837183+0000", 2024-09-21T10:24:28.824 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837183+0000", 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837182+0000", 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837188+0000", 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.874e-06 2024-09-21T10:24:28.825 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837203+0000", 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5675000000000002e-05 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837212+0000", 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7110000000000001e-06 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.826 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837265+0000", 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3266000000000001e-05 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837274+0000", 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4370000000000007e-06 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817384+0000", 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98011053000000004 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.827 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817458+0000", 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3233999999999997e-05 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817568+0000", 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000109917 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827010+0000", 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094422699999999991 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.828 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827015+0000", 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2279999999999998e-06 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827047+0000", 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1959999999999999e-05 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827082+0000", 2024-09-21T10:24:28.829 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5169000000000002e-05 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827102+0000", 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9758e-05 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:24:28.830 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:24:28.856 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:28.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:28.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:28 smithi032 ceph-mon[101299]: 1.3 scrub starts 2024-09-21T10:24:28.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:28 smithi032 ceph-mon[101299]: 1.3 scrub ok 2024-09-21T10:24:28.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:28 smithi032 ceph-mon[105748]: 1.3 scrub starts 2024-09-21T10:24:28.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:28 smithi032 ceph-mon[105748]: 1.3 scrub ok 2024-09-21T10:24:28.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:28.980 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:28.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:28 smithi045 ceph-mon[95976]: 1.3 scrub starts 2024-09-21T10:24:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:28 smithi045 ceph-mon[95976]: 1.3 scrub ok 2024-09-21T10:24:29.081 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:29.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:29.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:29.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:29.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:29.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:29.282 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:24:29.330 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:29.352 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:29.454 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:29.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:29.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:29.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:29.601 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:29.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:29.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:29.725 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:29.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:29.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:24:29.789 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:24:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:24:29] "GET /metrics HTTP/1.1" 200 34148 "" "Prometheus/2.51.0" 2024-09-21T10:24:29.826 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:29.848 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:29.950 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:30.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.004 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:30.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:29 smithi045 ceph-mon[95976]: pgmap v1138: 99 pgs: 9 creating+peering, 4 unknown, 1 peering, 85 active+clean; 4.4 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s rd, 93 KiB/s wr, 23 op/s; 245 KiB/s, 0 objects/s recovering 2024-09-21T10:24:30.051 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:29 smithi032 ceph-mon[105748]: pgmap v1138: 99 pgs: 9 creating+peering, 4 unknown, 1 peering, 85 active+clean; 4.4 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s rd, 93 KiB/s wr, 23 op/s; 245 KiB/s, 0 objects/s recovering 2024-09-21T10:24:30.051 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:29 smithi032 ceph-mon[101299]: pgmap v1138: 99 pgs: 9 creating+peering, 4 unknown, 1 peering, 85 active+clean; 4.4 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s rd, 93 KiB/s wr, 23 op/s; 245 KiB/s, 0 objects/s recovering 2024-09-21T10:24:30.105 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.129 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:30.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:30.230 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:30.284 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:30.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:30.386 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:30.409 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:30.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:30.511 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:30.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:30.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:30.641 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:30.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:30.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:30.750 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:24:30.767 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:30.823 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:30.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:30.844 INFO:teuthology.orchestra.run.smithi032.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-09-21T10:24:30.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.953 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:30.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:31.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:31.077 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:31.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:31.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:31.205 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:31.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:30 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/171452511' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:24:31.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:30 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/171452511' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:24:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:30 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/171452511' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:24:31.307 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:31.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:31.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:31.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:31.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.460 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:31.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:31.561 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:31.585 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:31.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:31.687 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:31.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:31.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:31.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:31.835 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:31.852 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:24:31.937 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:31.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.963 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:31.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:32.058 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[105748]: pgmap v1139: 99 pgs: 99 active+clean; 4.5 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:24:32.058 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/171452511' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:24:32.058 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[105748]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T10:24:32.058 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[105748]: 1.1 scrub starts 2024-09-21T10:24:32.058 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[105748]: 1.1 scrub ok 2024-09-21T10:24:32.058 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3053635305' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:32.058 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:32.058 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[101299]: pgmap v1139: 99 pgs: 99 active+clean; 4.5 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:24:32.058 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/171452511' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:24:32.059 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[101299]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T10:24:32.059 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[101299]: 1.1 scrub starts 2024-09-21T10:24:32.059 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[101299]: 1.1 scrub ok 2024-09-21T10:24:32.059 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3053635305' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:32.059 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:31 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:32.063 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:32.090 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:32.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:32.192 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:32.217 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:32.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:31 smithi045 ceph-mon[95976]: pgmap v1139: 99 pgs: 99 active+clean; 4.5 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:24:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:31 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/171452511' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:24:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:31 smithi045 ceph-mon[95976]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T10:24:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:31 smithi045 ceph-mon[95976]: 1.1 scrub starts 2024-09-21T10:24:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:31 smithi045 ceph-mon[95976]: 1.1 scrub ok 2024-09-21T10:24:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:31 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3053635305' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:31 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:32.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:32.342 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:32.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:32.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:32.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:32.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:32.596 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:32.698 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:32.724 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:32.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:32.739 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:32.740 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:32.756 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:32.826 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:32.853 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:32.955 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:32.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:32 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:32.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:32 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:24:32.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:32 smithi032 ceph-mon[101299]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T10:24:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:32 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:32 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:24:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:32 smithi032 ceph-mon[105748]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T10:24:32.978 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:32.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:33.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:33.082 DEBUG:teuthology.orchestra.run.smithi032:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-09-21T10:24:33.085 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:24:33.085 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:24:33.107 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:33.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:33.209 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:33.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:32 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:32 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:24:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:32 smithi045 ceph-mon[95976]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T10:24:33.315 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:hints = 1 2024-09-21T10:24:33.315 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:24:33.315 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object prefix: benchmark_data_smithi032_265250 2024-09-21T10:24:33.315 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:24:33.316 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:24:33.333 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:33.377 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:33.479 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:33.513 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:33.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:33.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:33.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.657 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:33.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:33.757 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:33.792 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:33.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:33.894 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.935 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:33.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:34.036 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:34.070 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:34.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:34.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:33 smithi032 ceph-mon[105748]: pgmap v1142: 99 pgs: 99 active+clean; 4.5 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:24:34.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:33 smithi032 ceph-mon[105748]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:24:34.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:34.117 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:34.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:33 smithi032 ceph-mon[101299]: pgmap v1142: 99 pgs: 99 active+clean; 4.5 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:24:34.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:33 smithi032 ceph-mon[101299]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:24:34.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:34.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:34.171 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:34.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:33 smithi045 ceph-mon[95976]: pgmap v1142: 99 pgs: 99 active+clean; 4.5 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:24:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:33 smithi045 ceph-mon[95976]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:24:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:34.311 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:34.316 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 1 16 3755 3739 233.675 233.688 0.00335416 0.00426394 2024-09-21T10:24:34.348 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:34.450 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:34.486 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:34.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:34.588 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:34.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.627 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:34.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:34.727 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:24:34.728 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:34.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:34.865 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:34.912 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:34.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:35.014 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:35.051 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:35.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:35.190 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:35.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:35.292 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:35.315 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 2 16 7191 7175 224.201 214.75 0.00481604 0.00445012 2024-09-21T10:24:35.326 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:35.428 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:35.469 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:35.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:35.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:35.610 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:35.712 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:35.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:35.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:35.885 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:35.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:35.910 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:35 smithi032 ceph-mon[105748]: pgmap v1143: 99 pgs: 99 active+clean; 40 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 4.5 MiB/s wr, 64 op/s 2024-09-21T10:24:35.910 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:35 smithi032 ceph-mon[101299]: pgmap v1143: 99 pgs: 99 active+clean; 40 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 4.5 MiB/s wr, 64 op/s 2024-09-21T10:24:35.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:36.027 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:36.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:36.129 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:36.169 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:36.271 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:35 smithi045 ceph-mon[95976]: pgmap v1143: 99 pgs: 99 active+clean; 40 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 4.5 MiB/s wr, 64 op/s 2024-09-21T10:24:36.309 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:36.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:36.316 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 3 16 10598 10582 220.44 212.938 0.0048207 0.00452886 2024-09-21T10:24:36.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:36.444 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:36.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:36.546 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:36.583 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:36.685 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:36.719 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:36.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:36.820 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:36.855 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:36.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:36.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:36.993 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:36.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:37.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:37.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:37.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:37.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:37.266 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:37.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:37.317 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 4 16 13546 13530 211.328 184.25 0.0042611 0.00472338 2024-09-21T10:24:37.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:37.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.412 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:37.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:37.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:37.562 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:37.665 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:37.706 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:37.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:37.808 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:37.845 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:37.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:37.947 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:37.983 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:37.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:38.085 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:38.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:37 smithi032 ceph-mon[101299]: pgmap v1144: 99 pgs: 99 active+clean; 275 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 34 MiB/s wr, 529 op/s 2024-09-21T10:24:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:37 smithi032 ceph-mon[105748]: pgmap v1144: 99 pgs: 99 active+clean; 275 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 34 MiB/s wr, 529 op/s 2024-09-21T10:24:38.211 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:38.246 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:38.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:37 smithi045 ceph-mon[95976]: pgmap v1144: 99 pgs: 99 active+clean; 275 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 34 MiB/s wr, 529 op/s 2024-09-21T10:24:38.317 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 5 16 16041 16025 200.249 155.938 0.0055797 0.00498844 2024-09-21T10:24:38.348 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:38.386 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:38.488 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:38.531 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:38.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:38.633 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:38.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.667 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:38.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:38.768 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:38.800 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:38.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:38.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:38.945 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45848.0:17752 9.b 9:d2a775a9:::benchmark_data_smithi032_265250_object17751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:24:38.941733+0000", 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.000141194, 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000177758, 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45848", 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:24:38.946 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 17752 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:24:38.941733+0000", 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:24:38.941733+0000", 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:24:38.947 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:24:38.941735+0000", 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5960000000000001e-06 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:24:38.941761+0000", 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6169e-05 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:24:38.941781+0000", 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0296999999999999e-05 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:24:38.948 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:24:38.941791+0000", 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0063e-05 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:24:38.941868+0000", 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6804999999999994e-05 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 1 2024-09-21T10:24:38.949 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:24:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:38.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:39.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:39.089 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:39.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:39.191 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:39.225 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:39.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:39.317 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 6 16 18621 18605 193.748 161.25 0.00649747 0.00515569 2024-09-21T10:24:39.327 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:39.363 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:39.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:39.464 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:39.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:39.603 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:39.627 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:39.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:39.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:39.767 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:39.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:39.869 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:39.875 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:24:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:24:39] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.51.0" 2024-09-21T10:24:39.905 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:40.007 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:40.041 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:40.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:40.143 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:40.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:40.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:40.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:39 smithi032 ceph-mon[105748]: pgmap v1145: 99 pgs: 99 active+clean; 414 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2024-09-21T10:24:40.192 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:39 smithi032 ceph-mon[101299]: pgmap v1145: 99 pgs: 99 active+clean; 414 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2024-09-21T10:24:40.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:39 smithi045 ceph-mon[95976]: pgmap v1145: 99 pgs: 99 active+clean; 414 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2024-09-21T10:24:40.318 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 7 16 21086 21070 188.077 154.062 0.00543836 0.00531005 2024-09-21T10:24:40.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.327 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:40.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:40.414 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:40.428 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:40.462 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:40.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:40.564 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:40.608 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:40.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:40.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:40.781 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:40.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:40.883 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:40.915 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:40.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:41.017 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:41.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:41.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:41.049 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:41 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:41.049 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:41 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:41.049 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:41 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:41.049 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:41 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:41.149 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:41.194 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:41.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:41.255 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:24:41.255 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:24:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:41 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:41 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:41.291 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:24:41.291 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:24:41.297 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:41.317 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 8 16 23328 23312 182.083 140.125 0.0069291 0.00548398 2024-09-21T10:24:41.328 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:24:41.328 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:24:41.335 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:41.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:41.376 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:24:41.377 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:24:41.411 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:24:41.411 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:24:41.437 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:41.448 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:24:41.448 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:24:41.475 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:41.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:41.484 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:24:41.484 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:24:41.519 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:24:41.519 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:24:41.577 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:41.617 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:41.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:41.719 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:41.741 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:41.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:41.876 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:41.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:41.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:42.015 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:42.117 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:42.140 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:42.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:42.242 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:42.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:42.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:42.283 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[101299]: pgmap v1146: 99 pgs: 99 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T10:24:42.283 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:24:42.283 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:24:42.284 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:42.284 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1166097706' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:42.284 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[105748]: pgmap v1146: 99 pgs: 99 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T10:24:42.284 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:24:42.284 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:24:42.284 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:42.284 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:42 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1166097706' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:42.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:42 smithi045 ceph-mon[95976]: pgmap v1146: 99 pgs: 99 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T10:24:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:42 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:24:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:42 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:24:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:42 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:24:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:42 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1166097706' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:42.317 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 9 16 25173 25157 174.663 115.312 0.00162023 0.00567494 2024-09-21T10:24:42.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:42.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.403 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:42.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:42.504 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.531 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:42.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:42.632 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:42.669 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:42.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:42.772 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:42.805 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:24:42.806 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:24:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.808 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:42.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:42.909 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:42.935 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:43.038 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:43.182 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:43.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:43 smithi032 ceph-mon[105748]: from='client.35934 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:43.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:43 smithi032 ceph-mon[101299]: from='client.35934 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:43.216 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:43.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:43.317 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 10 16 27087 27071 169.159 119.625 0.0133474 0.00590608 2024-09-21T10:24:43.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:43.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:43.460 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:43.482 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:43.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:43.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:43 smithi045 ceph-mon[95976]: from='client.35934 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:43.584 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:43.623 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:43.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:43.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:43.761 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:43.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:43.900 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:44.002 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:44.043 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:44.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:44.145 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:44.185 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:44.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:44.287 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:44.317 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 11 16 29135 29119 165.417 128 0.00565884 0.00603988 2024-09-21T10:24:44.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.329 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:44.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:44.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:44 smithi032 ceph-mon[101299]: pgmap v1147: 99 pgs: 99 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-21T10:24:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:44 smithi032 ceph-mon[105748]: pgmap v1147: 99 pgs: 99 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-21T10:24:44.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:44.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:44 smithi045 ceph-mon[95976]: pgmap v1147: 99 pgs: 99 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-21T10:24:44.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:44.601 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:44.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:44.703 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:44.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:44.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:44.843 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:44.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:44.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:45.022 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:45.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:45.124 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:45.159 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:45.261 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:45.298 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:45.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:45.318 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 12 16 31251 31235 162.652 132.25 0.006771 0.00614291 2024-09-21T10:24:45.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:45.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.438 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:45.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:45.539 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:45.563 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:45.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:45.665 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:45.700 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:45.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:45.802 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:45.842 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:45.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:45.944 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:45.980 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:45.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:46.082 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:46.122 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:46.224 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:46.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:46.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:46.318 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 13 16 33175 33159 159.39 120.25 0.00206664 0.00626416 2024-09-21T10:24:46.361 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:46.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.396 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:46.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:46 smithi032 ceph-mon[101299]: pgmap v1148: 99 pgs: 99 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-21T10:24:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:46 smithi032 ceph-mon[105748]: pgmap v1148: 99 pgs: 99 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-21T10:24:46.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:46.532 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:46.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:46.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:46 smithi045 ceph-mon[95976]: pgmap v1148: 99 pgs: 99 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-21T10:24:46.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:46.669 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:46.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:46.771 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:46.807 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:46.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:46.909 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:46.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:47.052 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:47.088 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:47.190 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:47.226 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:47.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:47.318 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 14 16 35053 35037 156.388 117.375 0.00512919 0.00638908 2024-09-21T10:24:47.327 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:47.365 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:47.467 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:47.502 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:47.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:47.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:47.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:47.743 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:47.778 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:47.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:47.880 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:47.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:48.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:48.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.059 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:48.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:48.160 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:48.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:48.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:48.297 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:48.300 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:48 smithi032 ceph-mon[101299]: pgmap v1149: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-21T10:24:48.300 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:48 smithi032 ceph-mon[105748]: pgmap v1149: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-21T10:24:48.319 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 15 16 37141 37125 154.653 130.5 0.00962622 0.00646076 2024-09-21T10:24:48.336 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:48 smithi045 ceph-mon[95976]: pgmap v1149: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-21T10:24:48.341 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:48.443 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:48.479 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:48.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:48.581 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:48.618 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:48.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:48.721 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:48.757 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:48.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:48.794 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:24:48.859 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:48.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.919 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:48.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:49.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:49.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.063 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:49.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:49.164 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:49.201 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:49.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:49.303 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:49.319 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 16 16 39082 39066 152.569 121.312 0.00976301 0.00654871 2024-09-21T10:24:49.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:49.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:49.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:49.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:49.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:49.477 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:49.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:49.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:49.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:49.613 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:49.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:49.715 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:49.750 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:49.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:49.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:49.865 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:24:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:24:49] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.51.0" 2024-09-21T10:24:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.895 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:49.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:49.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:50.035 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:50.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:50.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:24:50.131 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:50 smithi032 ceph-mon[101299]: pgmap v1150: 99 pgs: 99 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-09-21T10:24:50.131 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:50 smithi032 ceph-mon[105748]: pgmap v1150: 99 pgs: 99 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-09-21T10:24:50.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:50.173 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:50.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:50.233 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1146,"stamp":"2024-09-21T10:24:40.561771+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1207548176,"num_objects":18382,"num_object_clones":0,"num_object_copies":55146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18382,"num_whiteouts":0,"num_read":6066,"num_read_kb":13160,"num_write":19301,"num_write_kb":1187783,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":291,"num_bytes_recovered":17738297,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21029,"ondisk_log_size":21029,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8633292,"kb_used_data":3887180,"kb_used_omap":40,"kb_used_meta":4746007,"kb_avail":741327924,"statfs":{"total":767960285184,"available":759119794176,"internally_reserved":0,"allocated":3980472320,"data_stored":3964643551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41824,"internal_metadata":4859911328},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,6,9],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1202859610,"num_objects":18354,"num_object_clones":0,"num_object_copies":55062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18354,"num_whiteouts":0,"num_read":-36,"num_read_kb":-76,"num_write":18335,"num_write_kb":1174500,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-268350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.718976"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":641,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849418+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:24:31.849418+0000","last_peered":"2024-09-21T10:24:31.849418+0000","last_clean":"2024-09-21T10:24:31.849418+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:24:31.849418+0000","last_undegraded":"2024-09-21T10:24:31.849418+0000","last_fullsized":"2024-09-21T10:24:31.849418+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":385,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844455+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:24:30.844455+0000","last_peered":"2024-09-21T10:24:30.844455+0000","last_clean":"2024-09-21T10:24:30.844455+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:24:30.844455+0000","last_undegraded":"2024-09-21T10:24:30.844455+0000","last_fullsized":"2024-09-21T10:24:30.844455+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":81,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844835+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:24:30.844835+0000","last_peered":"2024-09-21T10:24:30.844835+0000","last_clean":"2024-09-21T10:24:30.844835+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:24:30.844835+0000","last_undegraded":"2024-09-21T10:24:30.844835+0000","last_fullsized":"2024-09-21T10:24:30.844835+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":83,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062499+0000","last_change":"2024-09-21T10:22:01.318638+0000","last_active":"2024-09-21T10:24:32.062499+0000","last_peered":"2024-09-21T10:24:32.062499+0000","last_clean":"2024-09-21T10:24:32.062499+0000","last_became_active":"2024-09-21T10:22:01.317087+0000","last_became_peered":"2024-09-21T10:22:01.317087+0000","last_unstale":"2024-09-21T10:24:32.062499+0000","last_undegraded":"2024-09-21T10:24:32.062499+0000","last_fullsized":"2024-09-21T10:24:32.062499+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:25.334726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":84,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062731+0000","last_change":"2024-09-21T10:22:01.318978+0000","last_active":"2024-09-21T10:24:32.062731+0000","last_peered":"2024-09-21T10:24:32.062731+0000","last_clean":"2024-09-21T10:24:32.062731+0000","last_became_active":"2024-09-21T10:22:01.318255+0000","last_became_peered":"2024-09-21T10:22:01.318255+0000","last_unstale":"2024-09-21T10:24:32.062731+0000","last_undegraded":"2024-09-21T10:24:32.062731+0000","last_fullsized":"2024-09-21T10:24:32.062731+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:55.084088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":356,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844312+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:24:30.844312+0000","last_peered":"2024-09-21T10:24:30.844312+0000","last_clean":"2024-09-21T10:24:30.844312+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:24:30.844312+0000","last_undegraded":"2024-09-21T10:24:30.844312+0000","last_fullsized":"2024-09-21T10:24:30.844312+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":662,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849667+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:24:31.849667+0000","last_peered":"2024-09-21T10:24:31.849667+0000","last_clean":"2024-09-21T10:24:31.849667+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:24:31.849667+0000","last_undegraded":"2024-09-21T10:24:31.849667+0000","last_fullsized":"2024-09-21T10:24:31.849667+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":437,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036345+0000","last_change":"2024-09-21T10:22:01.331612+0000","last_active":"2024-09-21T10:24:31.036345+0000","last_peered":"2024-09-21T10:24:31.036345+0000","last_clean":"2024-09-21T10:24:31.036345+0000","last_became_active":"2024-09-21T10:22:01.325282+0000","last_became_peered":"2024-09-21T10:22:01.325282+0000","last_unstale":"2024-09-21T10:24:31.036345+0000","last_undegraded":"2024-09-21T10:24:31.036345+0000","last_fullsized":"2024-09-21T10:24:31.036345+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:23.383106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"382'722","reported_seq":740,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:36.713056+0000","last_change":"2024-09-21T10:24:25.833692+0000","last_active":"2024-09-21T10:24:36.713056+0000","last_peered":"2024-09-21T10:24:36.713056+0000","last_clean":"2024-09-21T10:24:36.713056+0000","last_became_active":"2024-09-21T10:24:25.833390+0000","last_became_peered":"2024-09-21T10:24:25.833390+0000","last_unstale":"2024-09-21T10:24:36.713056+0000","last_undegraded":"2024-09-21T10:24:36.713056+0000","last_fullsized":"2024-09-21T10:24:36.713056+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":722,"log_dups_size":0,"ondisk_log_size":722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:12.227411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47316992,"num_objects":722,"num_object_clones":0,"num_object_copies":2166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":722,"num_write_kb":46208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":72,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844815+0000","last_change":"2024-09-21T10:22:01.321467+0000","last_active":"2024-09-21T10:24:30.844815+0000","last_peered":"2024-09-21T10:24:30.844815+0000","last_clean":"2024-09-21T10:24:30.844815+0000","last_became_active":"2024-09-21T10:22:01.318270+0000","last_became_peered":"2024-09-21T10:22:01.318270+0000","last_unstale":"2024-09-21T10:24:30.844815+0000","last_undegraded":"2024-09-21T10:24:30.844815+0000","last_fullsized":"2024-09-21T10:24:30.844815+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:44.059553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":90,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.843764+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:24:30.843764+0000","last_peered":"2024-09-21T10:24:30.843764+0000","last_clean":"2024-09-21T10:24:30.843764+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:24:30.843764+0000","last_undegraded":"2024-09-21T10:24:30.843764+0000","last_fullsized":"2024-09-21T10:24:30.843764+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"382'1361","reported_seq":1380,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:40.500916+0000","last_change":"2024-09-21T10:24:25.829780+0000","last_active":"2024-09-21T10:24:40.500916+0000","last_peered":"2024-09-21T10:24:40.500916+0000","last_clean":"2024-09-21T10:24:40.500916+0000","last_became_active":"2024-09-21T10:24:25.829509+0000","last_became_peered":"2024-09-21T10:24:25.829509+0000","last_unstale":"2024-09-21T10:24:40.500916+0000","last_undegraded":"2024-09-21T10:24:40.500916+0000","last_fullsized":"2024-09-21T10:24:40.500916+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1361,"log_dups_size":0,"ondisk_log_size":1361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:30.108791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89194496,"num_objects":1361,"num_object_clones":0,"num_object_copies":4083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1361,"num_write_kb":87104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":59,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844925+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:24:30.844925+0000","last_peered":"2024-09-21T10:24:30.844925+0000","last_clean":"2024-09-21T10:24:30.844925+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:24:30.844925+0000","last_undegraded":"2024-09-21T10:24:30.844925+0000","last_fullsized":"2024-09-21T10:24:30.844925+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036076+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:24:31.036076+0000","last_peered":"2024-09-21T10:24:31.036076+0000","last_clean":"2024-09-21T10:24:31.036076+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:24:31.036076+0000","last_undegraded":"2024-09-21T10:24:31.036076+0000","last_fullsized":"2024-09-21T10:24:31.036076+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"382'1259","reported_seq":1272,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:40.138121+0000","last_change":"2024-09-21T10:24:25.825494+0000","last_active":"2024-09-21T10:24:40.138121+0000","last_peered":"2024-09-21T10:24:40.138121+0000","last_clean":"2024-09-21T10:24:40.138121+0000","last_became_active":"2024-09-21T10:24:25.825196+0000","last_became_peered":"2024-09-21T10:24:25.825196+0000","last_unstale":"2024-09-21T10:24:40.138121+0000","last_undegraded":"2024-09-21T10:24:40.138121+0000","last_fullsized":"2024-09-21T10:24:40.138121+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1259,"log_dups_size":0,"ondisk_log_size":1259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:50.985537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82509824,"num_objects":1259,"num_object_clones":0,"num_object_copies":3777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1259,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":381,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.845042+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:24:30.845042+0000","last_peered":"2024-09-21T10:24:30.845042+0000","last_clean":"2024-09-21T10:24:30.845042+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:24:30.845042+0000","last_undegraded":"2024-09-21T10:24:30.845042+0000","last_fullsized":"2024-09-21T10:24:30.845042+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":400,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036062+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:24:31.036062+0000","last_peered":"2024-09-21T10:24:31.036062+0000","last_clean":"2024-09-21T10:24:31.036062+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:24:31.036062+0000","last_undegraded":"2024-09-21T10:24:31.036062+0000","last_fullsized":"2024-09-21T10:24:31.036062+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"382'1279","reported_seq":1292,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:40.139548+0000","last_change":"2024-09-21T10:24:25.846770+0000","last_active":"2024-09-21T10:24:40.139548+0000","last_peered":"2024-09-21T10:24:40.139548+0000","last_clean":"2024-09-21T10:24:40.139548+0000","last_became_active":"2024-09-21T10:24:25.846530+0000","last_became_peered":"2024-09-21T10:24:25.846530+0000","last_unstale":"2024-09-21T10:24:40.139548+0000","last_undegraded":"2024-09-21T10:24:40.139548+0000","last_fullsized":"2024-09-21T10:24:40.139548+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1279,"log_dups_size":0,"ondisk_log_size":1279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:50.827496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83820544,"num_objects":1279,"num_object_clones":0,"num_object_copies":3837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1279,"num_write_kb":81856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":315,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849614+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:24:31.849614+0000","last_peered":"2024-09-21T10:24:31.849614+0000","last_clean":"2024-09-21T10:24:31.849614+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:24:31.849614+0000","last_undegraded":"2024-09-21T10:24:31.849614+0000","last_fullsized":"2024-09-21T10:24:31.849614+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":253,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062326+0000","last_change":"2024-09-21T10:19:50.585724+0000","last_active":"2024-09-21T10:24:32.062326+0000","last_peered":"2024-09-21T10:24:32.062326+0000","last_clean":"2024-09-21T10:24:32.062326+0000","last_became_active":"2024-09-21T10:19:50.585315+0000","last_became_peered":"2024-09-21T10:19:50.585315+0000","last_unstale":"2024-09-21T10:24:32.062326+0000","last_undegraded":"2024-09-21T10:24:32.062326+0000","last_fullsized":"2024-09-21T10:24:32.062326+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:26.618970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"382'1138","reported_seq":1157,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:38.925409+0000","last_change":"2024-09-21T10:24:25.846977+0000","last_active":"2024-09-21T10:24:38.925409+0000","last_peered":"2024-09-21T10:24:38.925409+0000","last_clean":"2024-09-21T10:24:38.925409+0000","last_became_active":"2024-09-21T10:24:25.846670+0000","last_became_peered":"2024-09-21T10:24:25.846670+0000","last_unstale":"2024-09-21T10:24:38.925409+0000","last_undegraded":"2024-09-21T10:24:38.925409+0000","last_fullsized":"2024-09-21T10:24:38.925409+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1138,"log_dups_size":0,"ondisk_log_size":1138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:28.387249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74579968,"num_objects":1138,"num_object_clones":0,"num_object_copies":3414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1138,"num_write_kb":72832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":666,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844167+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:24:30.844167+0000","last_peered":"2024-09-21T10:24:30.844167+0000","last_clean":"2024-09-21T10:24:30.844167+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:24:30.844167+0000","last_undegraded":"2024-09-21T10:24:30.844167+0000","last_fullsized":"2024-09-21T10:24:30.844167+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":692,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849373+0000","last_change":"2024-09-21T10:22:01.335303+0000","last_active":"2024-09-21T10:24:31.849373+0000","last_peered":"2024-09-21T10:24:31.849373+0000","last_clean":"2024-09-21T10:24:31.849373+0000","last_became_active":"2024-09-21T10:22:01.325298+0000","last_became_peered":"2024-09-21T10:22:01.325298+0000","last_unstale":"2024-09-21T10:24:31.849373+0000","last_undegraded":"2024-09-21T10:24:31.849373+0000","last_fullsized":"2024-09-21T10:24:31.849373+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:01.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036848+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:24:31.036848+0000","last_peered":"2024-09-21T10:24:31.036848+0000","last_clean":"2024-09-21T10:24:31.036848+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:24:31.036848+0000","last_undegraded":"2024-09-21T10:24:31.036848+0000","last_fullsized":"2024-09-21T10:24:31.036848+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"382'1400","reported_seq":1419,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:40.462847+0000","last_change":"2024-09-21T10:24:25.847167+0000","last_active":"2024-09-21T10:24:40.462847+0000","last_peered":"2024-09-21T10:24:40.462847+0000","last_clean":"2024-09-21T10:24:40.462847+0000","last_became_active":"2024-09-21T10:24:25.846878+0000","last_became_peered":"2024-09-21T10:24:25.846878+0000","last_unstale":"2024-09-21T10:24:40.462847+0000","last_undegraded":"2024-09-21T10:24:40.462847+0000","last_fullsized":"2024-09-21T10:24:40.462847+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1400,"log_dups_size":0,"ondisk_log_size":1400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:32.840244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91750400,"num_objects":1400,"num_object_clones":0,"num_object_copies":4200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1400,"num_write_kb":89600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":427,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.843727+0000","last_change":"2024-09-21T10:24:24.830147+0000","last_active":"2024-09-21T10:24:30.843727+0000","last_peered":"2024-09-21T10:24:30.843727+0000","last_clean":"2024-09-21T10:24:30.843727+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:24:30.843727+0000","last_undegraded":"2024-09-21T10:24:30.843727+0000","last_fullsized":"2024-09-21T10:24:30.843727+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:24:21.271541+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":390,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036519+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:24:31.036519+0000","last_peered":"2024-09-21T10:24:31.036519+0000","last_clean":"2024-09-21T10:24:31.036519+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:24:31.036519+0000","last_undegraded":"2024-09-21T10:24:31.036519+0000","last_fullsized":"2024-09-21T10:24:31.036519+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.037149+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:24:31.037149+0000","last_peered":"2024-09-21T10:24:31.037149+0000","last_clean":"2024-09-21T10:24:31.037149+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:24:31.037149+0000","last_undegraded":"2024-09-21T10:24:31.037149+0000","last_fullsized":"2024-09-21T10:24:31.037149+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"382'1289","reported_seq":1302,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:39.917993+0000","last_change":"2024-09-21T10:24:25.828673+0000","last_active":"2024-09-21T10:24:39.917993+0000","last_peered":"2024-09-21T10:24:39.917993+0000","last_clean":"2024-09-21T10:24:39.917993+0000","last_became_active":"2024-09-21T10:24:25.828433+0000","last_became_peered":"2024-09-21T10:24:25.828433+0000","last_unstale":"2024-09-21T10:24:39.917993+0000","last_undegraded":"2024-09-21T10:24:39.917993+0000","last_fullsized":"2024-09-21T10:24:39.917993+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1289,"log_dups_size":0,"ondisk_log_size":1289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:07.654019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84475904,"num_objects":1289,"num_object_clones":0,"num_object_copies":3867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1289,"num_write_kb":82496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.035932+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:24:31.035932+0000","last_peered":"2024-09-21T10:24:31.035932+0000","last_clean":"2024-09-21T10:24:31.035932+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:24:31.035932+0000","last_undegraded":"2024-09-21T10:24:31.035932+0000","last_fullsized":"2024-09-21T10:24:31.035932+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.037178+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:24:31.037178+0000","last_peered":"2024-09-21T10:24:31.037178+0000","last_clean":"2024-09-21T10:24:31.037178+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:24:31.037178+0000","last_undegraded":"2024-09-21T10:24:31.037178+0000","last_fullsized":"2024-09-21T10:24:31.037178+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"382'1107","reported_seq":1126,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:38.919997+0000","last_change":"2024-09-21T10:24:25.847020+0000","last_active":"2024-09-21T10:24:38.919997+0000","last_peered":"2024-09-21T10:24:38.919997+0000","last_clean":"2024-09-21T10:24:38.919997+0000","last_became_active":"2024-09-21T10:24:25.846622+0000","last_became_peered":"2024-09-21T10:24:25.846622+0000","last_unstale":"2024-09-21T10:24:38.919997+0000","last_undegraded":"2024-09-21T10:24:38.919997+0000","last_fullsized":"2024-09-21T10:24:38.919997+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1107,"log_dups_size":0,"ondisk_log_size":1107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:54:04.668262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72548352,"num_objects":1107,"num_object_clones":0,"num_object_copies":3321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1107,"num_write_kb":70848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":605,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.843931+0000","last_change":"2024-09-21T10:24:24.825965+0000","last_active":"2024-09-21T10:24:30.843931+0000","last_peered":"2024-09-21T10:24:30.843931+0000","last_clean":"2024-09-21T10:24:30.843931+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:24:30.843931+0000","last_undegraded":"2024-09-21T10:24:30.843931+0000","last_fullsized":"2024-09-21T10:24:30.843931+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:22.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:22.489745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":670,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849088+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:24:31.849088+0000","last_peered":"2024-09-21T10:24:31.849088+0000","last_clean":"2024-09-21T10:24:31.849088+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:24:31.849088+0000","last_undegraded":"2024-09-21T10:24:31.849088+0000","last_fullsized":"2024-09-21T10:24:31.849088+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":363,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062898+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:24:32.062898+0000","last_peered":"2024-09-21T10:24:32.062898+0000","last_clean":"2024-09-21T10:24:32.062898+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:24:32.062898+0000","last_undegraded":"2024-09-21T10:24:32.062898+0000","last_fullsized":"2024-09-21T10:24:32.062898+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"382'1095","reported_seq":1114,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:38.909459+0000","last_change":"2024-09-21T10:24:25.832587+0000","last_active":"2024-09-21T10:24:38.909459+0000","last_peered":"2024-09-21T10:24:38.909459+0000","last_clean":"2024-09-21T10:24:38.909459+0000","last_became_active":"2024-09-21T10:24:25.832287+0000","last_became_peered":"2024-09-21T10:24:25.832287+0000","last_unstale":"2024-09-21T10:24:38.909459+0000","last_undegraded":"2024-09-21T10:24:38.909459+0000","last_fullsized":"2024-09-21T10:24:38.909459+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1095,"log_dups_size":0,"ondisk_log_size":1095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:03.975988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71761920,"num_objects":1095,"num_object_clones":0,"num_object_copies":3285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1095,"num_write_kb":70080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":650,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844099+0000","last_change":"2024-09-21T10:24:25.191654+0000","last_active":"2024-09-21T10:24:30.844099+0000","last_peered":"2024-09-21T10:24:30.844099+0000","last_clean":"2024-09-21T10:24:30.844099+0000","last_became_active":"2024-09-21T10:22:04.028938+0000","last_became_peered":"2024-09-21T10:22:04.028938+0000","last_unstale":"2024-09-21T10:24:30.844099+0000","last_undegraded":"2024-09-21T10:24:30.844099+0000","last_fullsized":"2024-09-21T10:24:30.844099+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:25.191555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:25.191555+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:28.455583+0000","scrub_duration":0.0015044590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":694,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844416+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:24:30.844416+0000","last_peered":"2024-09-21T10:24:30.844416+0000","last_clean":"2024-09-21T10:24:30.844416+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:24:30.844416+0000","last_undegraded":"2024-09-21T10:24:30.844416+0000","last_fullsized":"2024-09-21T10:24:30.844416+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":601,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849071+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:24:31.849071+0000","last_peered":"2024-09-21T10:24:31.849071+0000","last_clean":"2024-09-21T10:24:31.849071+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:24:31.849071+0000","last_undegraded":"2024-09-21T10:24:31.849071+0000","last_fullsized":"2024-09-21T10:24:31.849071+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":357,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036272+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:24:31.036272+0000","last_peered":"2024-09-21T10:24:31.036272+0000","last_clean":"2024-09-21T10:24:31.036272+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:24:31.036272+0000","last_undegraded":"2024-09-21T10:24:31.036272+0000","last_fullsized":"2024-09-21T10:24:31.036272+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"382'747","reported_seq":766,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:36.716217+0000","last_change":"2024-09-21T10:24:25.847325+0000","last_active":"2024-09-21T10:24:36.716217+0000","last_peered":"2024-09-21T10:24:36.716217+0000","last_clean":"2024-09-21T10:24:36.716217+0000","last_became_active":"2024-09-21T10:24:25.847024+0000","last_became_peered":"2024-09-21T10:24:25.847024+0000","last_unstale":"2024-09-21T10:24:36.716217+0000","last_undegraded":"2024-09-21T10:24:36.716217+0000","last_fullsized":"2024-09-21T10:24:36.716217+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":747,"log_dups_size":0,"ondisk_log_size":747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:43.595168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48955392,"num_objects":747,"num_object_clones":0,"num_object_copies":2241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":747,"num_write_kb":47808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":618,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.063193+0000","last_change":"2024-09-21T10:24:27.359111+0000","last_active":"2024-09-21T10:24:32.063193+0000","last_peered":"2024-09-21T10:24:32.063193+0000","last_clean":"2024-09-21T10:24:32.063193+0000","last_became_active":"2024-09-21T10:24:24.825850+0000","last_became_peered":"2024-09-21T10:24:24.825850+0000","last_unstale":"2024-09-21T10:24:32.063193+0000","last_undegraded":"2024-09-21T10:24:32.063193+0000","last_fullsized":"2024-09-21T10:24:32.063193+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:27.359065+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:24:27.359065+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:03.723058+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1022,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844183+0000","last_change":"2024-09-21T10:16:37.626232+0000","last_active":"2024-09-21T10:24:30.844183+0000","last_peered":"2024-09-21T10:24:30.844183+0000","last_clean":"2024-09-21T10:24:30.844183+0000","last_became_active":"2024-09-21T10:16:37.625210+0000","last_became_peered":"2024-09-21T10:16:37.625210+0000","last_unstale":"2024-09-21T10:24:30.844183+0000","last_undegraded":"2024-09-21T10:24:30.844183+0000","last_fullsized":"2024-09-21T10:24:30.844183+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:09.905520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":251,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062663+0000","last_change":"2024-09-21T10:19:51.555124+0000","last_active":"2024-09-21T10:24:32.062663+0000","last_peered":"2024-09-21T10:24:32.062663+0000","last_clean":"2024-09-21T10:24:32.062663+0000","last_became_active":"2024-09-21T10:19:51.554766+0000","last_became_peered":"2024-09-21T10:19:51.554766+0000","last_unstale":"2024-09-21T10:24:32.062663+0000","last_undegraded":"2024-09-21T10:24:32.062663+0000","last_fullsized":"2024-09-21T10:24:32.062663+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:52.832950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":83,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062524+0000","last_change":"2024-09-21T10:22:01.316140+0000","last_active":"2024-09-21T10:24:32.062524+0000","last_peered":"2024-09-21T10:24:32.062524+0000","last_clean":"2024-09-21T10:24:32.062524+0000","last_became_active":"2024-09-21T10:22:01.315408+0000","last_became_peered":"2024-09-21T10:22:01.315408+0000","last_unstale":"2024-09-21T10:24:32.062524+0000","last_undegraded":"2024-09-21T10:24:32.062524+0000","last_fullsized":"2024-09-21T10:24:32.062524+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:07.322279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"382'1276","reported_seq":1289,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:40.155351+0000","last_change":"2024-09-21T10:24:25.827798+0000","last_active":"2024-09-21T10:24:40.155351+0000","last_peered":"2024-09-21T10:24:40.155351+0000","last_clean":"2024-09-21T10:24:40.155351+0000","last_became_active":"2024-09-21T10:24:25.827230+0000","last_became_peered":"2024-09-21T10:24:25.827230+0000","last_unstale":"2024-09-21T10:24:40.155351+0000","last_undegraded":"2024-09-21T10:24:40.155351+0000","last_fullsized":"2024-09-21T10:24:40.155351+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1276,"log_dups_size":0,"ondisk_log_size":1276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:28:46.430403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83623936,"num_objects":1276,"num_object_clones":0,"num_object_copies":3828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1276,"num_write_kb":81664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":666,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844600+0000","last_change":"2024-09-21T10:24:24.829237+0000","last_active":"2024-09-21T10:24:30.844600+0000","last_peered":"2024-09-21T10:24:30.844600+0000","last_clean":"2024-09-21T10:24:30.844600+0000","last_became_active":"2024-09-21T10:24:24.828864+0000","last_became_peered":"2024-09-21T10:24:24.828864+0000","last_unstale":"2024-09-21T10:24:30.844600+0000","last_undegraded":"2024-09-21T10:24:30.844600+0000","last_fullsized":"2024-09-21T10:24:30.844600+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:33:22.286123+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1191,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.063251+0000","last_change":"2024-09-21T10:16:37.618380+0000","last_active":"2024-09-21T10:24:32.063251+0000","last_peered":"2024-09-21T10:24:32.063251+0000","last_clean":"2024-09-21T10:24:32.063251+0000","last_became_active":"2024-09-21T10:16:37.617829+0000","last_became_peered":"2024-09-21T10:16:37.617829+0000","last_unstale":"2024-09-21T10:24:32.063251+0000","last_undegraded":"2024-09-21T10:24:32.063251+0000","last_fullsized":"2024-09-21T10:24:32.063251+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:53.549786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":349,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062252+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:24:32.062252+0000","last_peered":"2024-09-21T10:24:32.062252+0000","last_clean":"2024-09-21T10:24:32.062252+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:24:32.062252+0000","last_undegraded":"2024-09-21T10:24:32.062252+0000","last_fullsized":"2024-09-21T10:24:32.062252+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":379,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.063037+0000","last_change":"2024-09-21T10:22:01.317933+0000","last_active":"2024-09-21T10:24:32.063037+0000","last_peered":"2024-09-21T10:24:32.063037+0000","last_clean":"2024-09-21T10:24:32.063037+0000","last_became_active":"2024-09-21T10:22:01.317536+0000","last_became_peered":"2024-09-21T10:22:01.317536+0000","last_unstale":"2024-09-21T10:24:32.063037+0000","last_undegraded":"2024-09-21T10:24:32.063037+0000","last_fullsized":"2024-09-21T10:24:32.063037+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:34.312724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"382'750","reported_seq":769,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:36.702285+0000","last_change":"2024-09-21T10:24:25.834043+0000","last_active":"2024-09-21T10:24:36.702285+0000","last_peered":"2024-09-21T10:24:36.702285+0000","last_clean":"2024-09-21T10:24:36.702285+0000","last_became_active":"2024-09-21T10:24:25.833657+0000","last_became_peered":"2024-09-21T10:24:25.833657+0000","last_unstale":"2024-09-21T10:24:36.702285+0000","last_undegraded":"2024-09-21T10:24:36.702285+0000","last_fullsized":"2024-09-21T10:24:36.702285+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":750,"log_dups_size":0,"ondisk_log_size":750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:06.262041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49152000,"num_objects":750,"num_object_clones":0,"num_object_copies":2250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":750,"num_write_kb":48000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":617,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.244931+0000","last_change":"2024-09-21T10:24:31.244931+0000","last_active":"2024-09-21T10:24:31.244931+0000","last_peered":"2024-09-21T10:24:31.244931+0000","last_clean":"2024-09-21T10:24:31.244931+0000","last_became_active":"2024-09-21T10:24:24.828799+0000","last_became_peered":"2024-09-21T10:24:24.828799+0000","last_unstale":"2024-09-21T10:24:31.244931+0000","last_undegraded":"2024-09-21T10:24:31.244931+0000","last_fullsized":"2024-09-21T10:24:31.244931+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:31.244809+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:31.244809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:02.931874+0000","scrub_duration":0.0015828649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1125,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844240+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:24:30.844240+0000","last_peered":"2024-09-21T10:24:30.844240+0000","last_clean":"2024-09-21T10:24:30.844240+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:24:30.844240+0000","last_undegraded":"2024-09-21T10:24:30.844240+0000","last_fullsized":"2024-09-21T10:24:30.844240+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036993+0000","last_change":"2024-09-21T10:19:50.581529+0000","last_active":"2024-09-21T10:24:31.036993+0000","last_peered":"2024-09-21T10:24:31.036993+0000","last_clean":"2024-09-21T10:24:31.036993+0000","last_became_active":"2024-09-21T10:19:50.581119+0000","last_became_peered":"2024-09-21T10:19:50.581119+0000","last_unstale":"2024-09-21T10:24:31.036993+0000","last_undegraded":"2024-09-21T10:24:31.036993+0000","last_fullsized":"2024-09-21T10:24:31.036993+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:13.334483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":349,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062648+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:24:32.062648+0000","last_peered":"2024-09-21T10:24:32.062648+0000","last_clean":"2024-09-21T10:24:32.062648+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:24:32.062648+0000","last_undegraded":"2024-09-21T10:24:32.062648+0000","last_fullsized":"2024-09-21T10:24:32.062648+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"382'1283","reported_seq":1293,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:40.161461+0000","last_change":"2024-09-21T10:24:25.831730+0000","last_active":"2024-09-21T10:24:40.161461+0000","last_peered":"2024-09-21T10:24:40.161461+0000","last_clean":"2024-09-21T10:24:40.161461+0000","last_became_active":"2024-09-21T10:24:25.831416+0000","last_became_peered":"2024-09-21T10:24:25.831416+0000","last_unstale":"2024-09-21T10:24:40.161461+0000","last_undegraded":"2024-09-21T10:24:40.161461+0000","last_fullsized":"2024-09-21T10:24:40.161461+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1283,"log_dups_size":0,"ondisk_log_size":1283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.763992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84082688,"num_objects":1283,"num_object_clones":0,"num_object_copies":3849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1283,"num_write_kb":82112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":465,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036512+0000","last_change":"2024-09-21T10:24:24.828950+0000","last_active":"2024-09-21T10:24:31.036512+0000","last_peered":"2024-09-21T10:24:31.036512+0000","last_clean":"2024-09-21T10:24:31.036512+0000","last_became_active":"2024-09-21T10:24:24.828550+0000","last_became_peered":"2024-09-21T10:24:24.828550+0000","last_unstale":"2024-09-21T10:24:31.036512+0000","last_undegraded":"2024-09-21T10:24:31.036512+0000","last_fullsized":"2024-09-21T10:24:31.036512+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:29.313198+0000","scrub_duration":0.001370488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":330,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844475+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:24:30.844475+0000","last_peered":"2024-09-21T10:24:30.844475+0000","last_clean":"2024-09-21T10:24:30.844475+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:24:30.844475+0000","last_undegraded":"2024-09-21T10:24:30.844475+0000","last_fullsized":"2024-09-21T10:24:30.844475+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":958,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062897+0000","last_change":"2024-09-21T10:16:37.618771+0000","last_active":"2024-09-21T10:24:32.062897+0000","last_peered":"2024-09-21T10:24:32.062897+0000","last_clean":"2024-09-21T10:24:32.062897+0000","last_became_active":"2024-09-21T10:16:37.618039+0000","last_became_peered":"2024-09-21T10:16:37.618039+0000","last_unstale":"2024-09-21T10:24:32.062897+0000","last_undegraded":"2024-09-21T10:24:32.062897+0000","last_fullsized":"2024-09-21T10:24:32.062897+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:42.158984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":385,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062836+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:24:32.062836+0000","last_peered":"2024-09-21T10:24:32.062836+0000","last_clean":"2024-09-21T10:24:32.062836+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:24:32.062836+0000","last_undegraded":"2024-09-21T10:24:32.062836+0000","last_fullsized":"2024-09-21T10:24:32.062836+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"382'1263","reported_seq":1276,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:39.910520+0000","last_change":"2024-09-21T10:24:25.831693+0000","last_active":"2024-09-21T10:24:39.910520+0000","last_peered":"2024-09-21T10:24:39.910520+0000","last_clean":"2024-09-21T10:24:39.910520+0000","last_became_active":"2024-09-21T10:24:25.831245+0000","last_became_peered":"2024-09-21T10:24:25.831245+0000","last_unstale":"2024-09-21T10:24:39.910520+0000","last_undegraded":"2024-09-21T10:24:39.910520+0000","last_fullsized":"2024-09-21T10:24:39.910520+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1263,"log_dups_size":0,"ondisk_log_size":1263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:30.943590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82771968,"num_objects":1263,"num_object_clones":0,"num_object_copies":3789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1263,"num_write_kb":80832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":465,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036805+0000","last_change":"2024-09-21T10:24:24.829059+0000","last_active":"2024-09-21T10:24:31.036805+0000","last_peered":"2024-09-21T10:24:31.036805+0000","last_clean":"2024-09-21T10:24:31.036805+0000","last_became_active":"2024-09-21T10:24:24.828647+0000","last_became_peered":"2024-09-21T10:24:24.828647+0000","last_unstale":"2024-09-21T10:24:31.036805+0000","last_undegraded":"2024-09-21T10:24:31.036805+0000","last_fullsized":"2024-09-21T10:24:31.036805+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:44:33.755670+0000","scrub_duration":0.0018106349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":367,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062800+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:24:32.062800+0000","last_peered":"2024-09-21T10:24:32.062800+0000","last_clean":"2024-09-21T10:24:32.062800+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:24:32.062800+0000","last_undegraded":"2024-09-21T10:24:32.062800+0000","last_fullsized":"2024-09-21T10:24:32.062800+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":948,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.845126+0000","last_change":"2024-09-21T10:22:01.336961+0000","last_active":"2024-09-21T10:24:30.845126+0000","last_peered":"2024-09-21T10:24:30.845126+0000","last_clean":"2024-09-21T10:24:30.845126+0000","last_became_active":"2024-09-21T10:22:01.320079+0000","last_became_peered":"2024-09-21T10:22:01.320079+0000","last_unstale":"2024-09-21T10:24:30.845126+0000","last_undegraded":"2024-09-21T10:24:30.845126+0000","last_fullsized":"2024-09-21T10:24:30.845126+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:54.037388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":250,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062224+0000","last_change":"2024-09-21T10:20:13.088557+0000","last_active":"2024-09-21T10:24:32.062224+0000","last_peered":"2024-09-21T10:24:32.062224+0000","last_clean":"2024-09-21T10:24:32.062224+0000","last_became_active":"2024-09-21T10:20:13.088123+0000","last_became_peered":"2024-09-21T10:20:13.088123+0000","last_unstale":"2024-09-21T10:24:32.062224+0000","last_undegraded":"2024-09-21T10:24:32.062224+0000","last_fullsized":"2024-09-21T10:24:32.062224+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:49.475499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"382'1271","reported_seq":1282,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:39.905438+0000","last_change":"2024-09-21T10:24:25.827432+0000","last_active":"2024-09-21T10:24:39.905438+0000","last_peered":"2024-09-21T10:24:39.905438+0000","last_clean":"2024-09-21T10:24:39.905438+0000","last_became_active":"2024-09-21T10:24:25.827047+0000","last_became_peered":"2024-09-21T10:24:25.827047+0000","last_unstale":"2024-09-21T10:24:39.905438+0000","last_undegraded":"2024-09-21T10:24:39.905438+0000","last_fullsized":"2024-09-21T10:24:39.905438+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1271,"log_dups_size":0,"ondisk_log_size":1271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:52.646714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1271,"num_write_kb":81344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":617,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.037197+0000","last_change":"2024-09-21T10:24:26.255840+0000","last_active":"2024-09-21T10:24:31.037197+0000","last_peered":"2024-09-21T10:24:31.037197+0000","last_clean":"2024-09-21T10:24:31.037197+0000","last_became_active":"2024-09-21T10:24:24.825549+0000","last_became_peered":"2024-09-21T10:24:24.825549+0000","last_unstale":"2024-09-21T10:24:31.037197+0000","last_undegraded":"2024-09-21T10:24:31.037197+0000","last_fullsized":"2024-09-21T10:24:31.037197+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:26.255753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.255753+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:25.638856+0000","scrub_duration":0.0016223839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":675,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849822+0000","last_change":"2024-09-21T10:21:34.876196+0000","last_active":"2024-09-21T10:24:31.849822+0000","last_peered":"2024-09-21T10:24:31.849822+0000","last_clean":"2024-09-21T10:24:31.849822+0000","last_became_active":"2024-09-21T10:21:34.868031+0000","last_became_peered":"2024-09-21T10:21:34.868031+0000","last_unstale":"2024-09-21T10:24:31.849822+0000","last_undegraded":"2024-09-21T10:24:31.849822+0000","last_fullsized":"2024-09-21T10:24:31.849822+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:58.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1021,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.063420+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:24:32.063420+0000","last_peered":"2024-09-21T10:24:32.063420+0000","last_clean":"2024-09-21T10:24:32.063420+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:24:32.063420+0000","last_undegraded":"2024-09-21T10:24:32.063420+0000","last_fullsized":"2024-09-21T10:24:32.063420+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":671,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.848878+0000","last_change":"2024-09-21T10:22:01.329400+0000","last_active":"2024-09-21T10:24:31.848878+0000","last_peered":"2024-09-21T10:24:31.848878+0000","last_clean":"2024-09-21T10:24:31.848878+0000","last_became_active":"2024-09-21T10:22:01.328887+0000","last_became_peered":"2024-09-21T10:22:01.328887+0000","last_unstale":"2024-09-21T10:24:31.848878+0000","last_undegraded":"2024-09-21T10:24:31.848878+0000","last_fullsized":"2024-09-21T10:24:31.848878+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:09.626637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"382'1115","reported_seq":1134,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:38.908673+0000","last_change":"2024-09-21T10:24:25.830908+0000","last_active":"2024-09-21T10:24:38.908673+0000","last_peered":"2024-09-21T10:24:38.908673+0000","last_clean":"2024-09-21T10:24:38.908673+0000","last_became_active":"2024-09-21T10:24:25.830171+0000","last_became_peered":"2024-09-21T10:24:25.830171+0000","last_unstale":"2024-09-21T10:24:38.908673+0000","last_undegraded":"2024-09-21T10:24:38.908673+0000","last_fullsized":"2024-09-21T10:24:38.908673+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":1115,"log_dups_size":0,"ondisk_log_size":1115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:00.292732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73072640,"num_objects":1115,"num_object_clones":0,"num_object_copies":3345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1115,"num_write_kb":71360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":654,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849212+0000","last_change":"2024-09-21T10:24:26.926717+0000","last_active":"2024-09-21T10:24:31.849212+0000","last_peered":"2024-09-21T10:24:31.849212+0000","last_clean":"2024-09-21T10:24:31.849212+0000","last_became_active":"2024-09-21T10:24:24.828205+0000","last_became_peered":"2024-09-21T10:24:24.828205+0000","last_unstale":"2024-09-21T10:24:31.849212+0000","last_undegraded":"2024-09-21T10:24:31.849212+0000","last_fullsized":"2024-09-21T10:24:31.849212+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:26.926661+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.926661+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:25.847900+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":87,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062620+0000","last_change":"2024-09-21T10:23:24.764361+0000","last_active":"2024-09-21T10:24:32.062620+0000","last_peered":"2024-09-21T10:24:32.062620+0000","last_clean":"2024-09-21T10:24:32.062620+0000","last_became_active":"2024-09-21T10:22:01.317312+0000","last_became_peered":"2024-09-21T10:22:01.317312+0000","last_unstale":"2024-09-21T10:24:32.062620+0000","last_undegraded":"2024-09-21T10:24:32.062620+0000","last_fullsized":"2024-09-21T10:24:32.062620+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:32:39.440581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":692,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036116+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:24:31.036116+0000","last_peered":"2024-09-21T10:24:31.036116+0000","last_clean":"2024-09-21T10:24:31.036116+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:24:31.036116+0000","last_undegraded":"2024-09-21T10:24:31.036116+0000","last_fullsized":"2024-09-21T10:24:31.036116+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":354,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036529+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:24:31.036529+0000","last_peered":"2024-09-21T10:24:31.036529+0000","last_clean":"2024-09-21T10:24:31.036529+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:24:31.036529+0000","last_undegraded":"2024-09-21T10:24:31.036529+0000","last_fullsized":"2024-09-21T10:24:31.036529+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":378,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.037026+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:24:31.037026+0000","last_peered":"2024-09-21T10:24:31.037026+0000","last_clean":"2024-09-21T10:24:31.037026+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:24:31.037026+0000","last_undegraded":"2024-09-21T10:24:31.037026+0000","last_fullsized":"2024-09-21T10:24:31.037026+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":88,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036974+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:24:31.036974+0000","last_peered":"2024-09-21T10:24:31.036974+0000","last_clean":"2024-09-21T10:24:31.036974+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:24:31.036974+0000","last_undegraded":"2024-09-21T10:24:31.036974+0000","last_fullsized":"2024-09-21T10:24:31.036974+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":81,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844149+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:24:30.844149+0000","last_peered":"2024-09-21T10:24:30.844149+0000","last_clean":"2024-09-21T10:24:30.844149+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:24:30.844149+0000","last_undegraded":"2024-09-21T10:24:30.844149+0000","last_fullsized":"2024-09-21T10:24:30.844149+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":61,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.845172+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:24:30.845172+0000","last_peered":"2024-09-21T10:24:30.845172+0000","last_clean":"2024-09-21T10:24:30.845172+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:24:30.845172+0000","last_undegraded":"2024-09-21T10:24:30.845172+0000","last_fullsized":"2024-09-21T10:24:30.845172+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844512+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:24:30.844512+0000","last_peered":"2024-09-21T10:24:30.844512+0000","last_clean":"2024-09-21T10:24:30.844512+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:24:30.844512+0000","last_undegraded":"2024-09-21T10:24:30.844512+0000","last_fullsized":"2024-09-21T10:24:30.844512+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":594,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849065+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:24:31.849065+0000","last_peered":"2024-09-21T10:24:31.849065+0000","last_clean":"2024-09-21T10:24:31.849065+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:24:31.849065+0000","last_undegraded":"2024-09-21T10:24:31.849065+0000","last_fullsized":"2024-09-21T10:24:31.849065+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":250,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062343+0000","last_change":"2024-09-21T10:20:13.089473+0000","last_active":"2024-09-21T10:24:32.062343+0000","last_peered":"2024-09-21T10:24:32.062343+0000","last_clean":"2024-09-21T10:24:32.062343+0000","last_became_active":"2024-09-21T10:20:13.089111+0000","last_became_peered":"2024-09-21T10:20:13.089111+0000","last_unstale":"2024-09-21T10:24:32.062343+0000","last_undegraded":"2024-09-21T10:24:32.062343+0000","last_fullsized":"2024-09-21T10:24:32.062343+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:06.603284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":654,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.848804+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:24:31.848804+0000","last_peered":"2024-09-21T10:24:31.848804+0000","last_clean":"2024-09-21T10:24:31.848804+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:24:31.848804+0000","last_undegraded":"2024-09-21T10:24:31.848804+0000","last_fullsized":"2024-09-21T10:24:31.848804+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":396,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036605+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:24:31.036605+0000","last_peered":"2024-09-21T10:24:31.036605+0000","last_clean":"2024-09-21T10:24:31.036605+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:24:31.036605+0000","last_undegraded":"2024-09-21T10:24:31.036605+0000","last_fullsized":"2024-09-21T10:24:31.036605+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":671,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.848994+0000","last_change":"2024-09-21T10:22:01.329690+0000","last_active":"2024-09-21T10:24:31.848994+0000","last_peered":"2024-09-21T10:24:31.848994+0000","last_clean":"2024-09-21T10:24:31.848994+0000","last_became_active":"2024-09-21T10:22:01.329033+0000","last_became_peered":"2024-09-21T10:22:01.329033+0000","last_unstale":"2024-09-21T10:24:31.848994+0000","last_undegraded":"2024-09-21T10:24:31.848994+0000","last_fullsized":"2024-09-21T10:24:31.848994+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:24.559524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":254,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062854+0000","last_change":"2024-09-21T10:21:34.870704+0000","last_active":"2024-09-21T10:24:32.062854+0000","last_peered":"2024-09-21T10:24:32.062854+0000","last_clean":"2024-09-21T10:24:32.062854+0000","last_became_active":"2024-09-21T10:21:34.869835+0000","last_became_peered":"2024-09-21T10:21:34.869835+0000","last_unstale":"2024-09-21T10:24:32.062854+0000","last_undegraded":"2024-09-21T10:24:32.062854+0000","last_fullsized":"2024-09-21T10:24:32.062854+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:34.959915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":369,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062242+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:24:32.062242+0000","last_peered":"2024-09-21T10:24:32.062242+0000","last_clean":"2024-09-21T10:24:32.062242+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:24:32.062242+0000","last_undegraded":"2024-09-21T10:24:32.062242+0000","last_fullsized":"2024-09-21T10:24:32.062242+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":661,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849065+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:24:31.849065+0000","last_peered":"2024-09-21T10:24:31.849065+0000","last_clean":"2024-09-21T10:24:31.849065+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:24:31.849065+0000","last_undegraded":"2024-09-21T10:24:31.849065+0000","last_fullsized":"2024-09-21T10:24:31.849065+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":385,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036862+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:24:31.036862+0000","last_peered":"2024-09-21T10:24:31.036862+0000","last_clean":"2024-09-21T10:24:31.036862+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:24:31.036862+0000","last_undegraded":"2024-09-21T10:24:31.036862+0000","last_fullsized":"2024-09-21T10:24:31.036862+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":660,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.063630+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:24:32.063630+0000","last_peered":"2024-09-21T10:24:32.063630+0000","last_clean":"2024-09-21T10:24:32.063630+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:24:32.063630+0000","last_undegraded":"2024-09-21T10:24:32.063630+0000","last_fullsized":"2024-09-21T10:24:32.063630+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":83,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062064+0000","last_change":"2024-09-21T10:22:01.318549+0000","last_active":"2024-09-21T10:24:32.062064+0000","last_peered":"2024-09-21T10:24:32.062064+0000","last_clean":"2024-09-21T10:24:32.062064+0000","last_became_active":"2024-09-21T10:22:01.318071+0000","last_became_peered":"2024-09-21T10:22:01.318071+0000","last_unstale":"2024-09-21T10:24:32.062064+0000","last_undegraded":"2024-09-21T10:24:32.062064+0000","last_fullsized":"2024-09-21T10:24:32.062064+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:49.061492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036863+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:24:31.036863+0000","last_peered":"2024-09-21T10:24:31.036863+0000","last_clean":"2024-09-21T10:24:31.036863+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:24:31.036863+0000","last_undegraded":"2024-09-21T10:24:31.036863+0000","last_fullsized":"2024-09-21T10:24:31.036863+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":321,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844224+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:24:30.844224+0000","last_peered":"2024-09-21T10:24:30.844224+0000","last_clean":"2024-09-21T10:24:30.844224+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:24:30.844224+0000","last_undegraded":"2024-09-21T10:24:30.844224+0000","last_fullsized":"2024-09-21T10:24:30.844224+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":77,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.036736+0000","last_change":"2024-09-21T10:19:50.588363+0000","last_active":"2024-09-21T10:24:31.036736+0000","last_peered":"2024-09-21T10:24:31.036736+0000","last_clean":"2024-09-21T10:24:31.036736+0000","last_became_active":"2024-09-21T10:19:50.588107+0000","last_became_peered":"2024-09-21T10:19:50.588107+0000","last_unstale":"2024-09-21T10:24:31.036736+0000","last_undegraded":"2024-09-21T10:24:31.036736+0000","last_fullsized":"2024-09-21T10:24:31.036736+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:05.698018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":299,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-21T10:24:30.844792+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:24:30.844792+0000","last_peered":"2024-09-21T10:24:30.844792+0000","last_clean":"2024-09-21T10:24:30.844792+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:24:30.844792+0000","last_undegraded":"2024-09-21T10:24:30.844792+0000","last_fullsized":"2024-09-21T10:24:30.844792+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":674,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849770+0000","last_change":"2024-09-21T10:16:35.248933+0000","last_active":"2024-09-21T10:24:31.849770+0000","last_peered":"2024-09-21T10:24:31.849770+0000","last_clean":"2024-09-21T10:24:31.849770+0000","last_became_active":"2024-09-21T10:16:35.248084+0000","last_became_peered":"2024-09-21T10:16:35.248084+0000","last_unstale":"2024-09-21T10:24:31.849770+0000","last_undegraded":"2024-09-21T10:24:31.849770+0000","last_fullsized":"2024-09-21T10:24:31.849770+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:00.978092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":636,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:31.849525+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:24:31.849525+0000","last_peered":"2024-09-21T10:24:31.849525+0000","last_clean":"2024-09-21T10:24:31.849525+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:24:31.849525+0000","last_undegraded":"2024-09-21T10:24:31.849525+0000","last_fullsized":"2024-09-21T10:24:31.849525+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":78,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062142+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:24:32.062142+0000","last_peered":"2024-09-21T10:24:32.062142+0000","last_clean":"2024-09-21T10:24:32.062142+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:24:32.062142+0000","last_undegraded":"2024-09-21T10:24:32.062142+0000","last_fullsized":"2024-09-21T10:24:32.062142+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":89,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-09-21T10:24:32.062169+0000","last_change":"2024-09-21T10:23:24.753979+0000","last_active":"2024-09-21T10:24:32.062169+0000","last_peered":"2024-09-21T10:24:32.062169+0000","last_clean":"2024-09-21T10:24:32.062169+0000","last_became_active":"2024-09-21T10:22:01.316569+0000","last_became_peered":"2024-09-21T10:22:01.316569+0000","last_unstale":"2024-09-21T10:24:32.062169+0000","last_undegraded":"2024-09-21T10:24:32.062169+0000","last_fullsized":"2024-09-21T10:24:32.062169+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:35:44.904752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1202913280,"num_objects":18355,"num_object_clones":0,"num_object_copies":55065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18355,"num_write_kb":1174720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3825664000,"data_stored":3825664000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18355,"ondisk_log_size":18355,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":6328,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24225,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":435019,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":69,"num_read_kb":150,"num_write":38,"num_write_kb":438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234420,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263516,"kb_used_data":650772,"kb_used_omap":1,"kb_used_meta":612734,"kb_avail":92481636,"statfs":{"total":95995035648,"available":94701195264,"internally_reserved":0,"allocated":666390528,"data_stored":664403635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":627440075},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496058,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1247768,"kb_used_data":616976,"kb_used_omap":1,"kb_used_meta":630782,"kb_avail":92497384,"statfs":{"total":95995035648,"available":94717321216,"internally_reserved":0,"allocated":631783424,"data_stored":629800831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":645921223},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403958,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":966356,"kb_used_data":675980,"kb_used_omap":7,"kb_used_meta":290360,"kb_avail":92778796,"statfs":{"total":95995035648,"available":95005487104,"internally_reserved":0,"allocated":692203520,"data_stored":690221393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":297329560},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019331,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":830300,"kb_used_data":225748,"kb_used_omap":1,"kb_used_meta":604542,"kb_avail":92914852,"statfs":{"total":95995035648,"available":95144808448,"internally_reserved":0,"allocated":231165952,"data_stored":229179320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":619051462},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435515,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1358604,"kb_used_data":568900,"kb_used_omap":5,"kb_used_meta":789690,"kb_avail":92386548,"statfs":{"total":95995035648,"available":94603825152,"internally_reserved":0,"allocated":582553600,"data_stored":580579096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":808642600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599042,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":916172,"kb_used_data":212548,"kb_used_omap":7,"kb_used_meta":703608,"kb_avail":92828980,"statfs":{"total":95995035648,"available":95056875520,"internally_reserved":0,"allocated":217649152,"data_stored":215674866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":720495053},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762569,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1403716,"kb_used_data":772284,"kb_used_omap":7,"kb_used_meta":631416,"kb_avail":92341436,"statfs":{"total":95995035648,"available":94557630464,"internally_reserved":0,"allocated":790818816,"data_stored":788851616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":646570237},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926095,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":646860,"kb_used_data":163972,"kb_used_omap":7,"kb_used_meta":482872,"kb_avail":93098292,"statfs":{"total":95995035648,"available":95332651008,"internally_reserved":0,"allocated":167907328,"data_stored":165932794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8002,"internal_metadata":494461118},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6408,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":193527808,"data_stored":193527808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":797048832,"data_stored":797048832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":237174784,"data_stored":237174784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":579403776,"data_stored":579403776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":255328256,"data_stored":255328256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":758775808,"data_stored":758775808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":259588096,"data_stored":259588096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":744816640,"data_stored":744816640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:24:50.237 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:24:50.237 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:24:50.237 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 1, 2, 3, 4, 6, 7, 5] 2024-09-21T10:24:50.237 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T10:24:50.237 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:24:50.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:50.321 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 17 16 40943 40927 150.436 116.312 0.0078654 0.00664204 2024-09-21T10:24:50.334 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:50.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:50.383 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:24:50 smithi032 systemd[1]: Stopping Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:24:50.410 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:50 smithi045 ceph-mon[95976]: pgmap v1150: 99 pgs: 99 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-09-21T10:24:50.437 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:50.469 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:50.571 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.595 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:50.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:50.696 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:50.728 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:50.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:50.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:50.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:50.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:50.932 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:50.943 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:51.006 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:51.049 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:51.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:51.152 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:51.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.193 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:51.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:51.294 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:51.319 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 18 16 42523 42507 147.562 98.75 0.00760576 0.00677072 2024-09-21T10:24:51.332 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:51.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:51.434 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:51.473 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:51.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:51.575 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:51.614 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:51.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:51.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:24:51 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[115573]: 2024-09-21T10:24:51.338+0000 7f9d97fe9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:24:51.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:24:51 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[115573]: 2024-09-21T10:24:51.338+0000 7f9d97fe9640 -1 osd.0 382 *** Got signal Terminated *** 2024-09-21T10:24:51.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:24:51 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[115573]: 2024-09-21T10:24:51.338+0000 7f9d97fe9640 -1 osd.0 382 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:24:51.715 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.751 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:51.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:51.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:51.890 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:51.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:51.992 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.035 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:52.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:52.090 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:24:52.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:52.159 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:52.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:52.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:52.282 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:52.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:52.319 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 19 16 44071 44055 144.887 96.75 0.00224099 0.00682354 2024-09-21T10:24:52.384 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:52.406 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:52.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:52 smithi032 ceph-mon[105748]: pgmap v1151: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 168 MiB/s wr, 2.69k op/s 2024-09-21T10:24:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:52 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3622417417' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:52 smithi032 ceph-mon[105748]: osd.0 marked itself down and dead 2024-09-21T10:24:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:52 smithi032 ceph-mon[101299]: pgmap v1151: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 168 MiB/s wr, 2.69k op/s 2024-09-21T10:24:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:52 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3622417417' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:52 smithi032 ceph-mon[101299]: osd.0 marked itself down and dead 2024-09-21T10:24:52.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:52.531 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:52.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:52.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:52 smithi045 ceph-mon[95976]: pgmap v1151: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 168 MiB/s wr, 2.69k op/s 2024-09-21T10:24:52.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:52 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3622417417' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:52.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:52 smithi045 ceph-mon[95976]: osd.0 marked itself down and dead 2024-09-21T10:24:52.632 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:52.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:52.758 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:52.781 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:52.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:52.882 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:52.905 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:52.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:53.007 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:53.029 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:53.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:53.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:53.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.166 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:53.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:53 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:53 smithi032 ceph-mon[101299]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:24:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:53 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3622417417' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:24:53.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:53 smithi032 ceph-mon[101299]: osdmap e383: 8 total, 7 up, 8 in 2024-09-21T10:24:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:53 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:53 smithi032 ceph-mon[105748]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:24:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:53 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3622417417' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:24:53.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:53 smithi032 ceph-mon[105748]: osdmap e383: 8 total, 7 up, 8 in 2024-09-21T10:24:53.208 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:24:52 smithi032 podman[267112]: 2024-09-21 10:24:52.887383336 +0000 UTC m=+1.947270151 container died 0e29e15aee7335c2b9bd21eae3f5e8bce7549f6cec4437eef84ee18f06e2e718 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-62da54b) 2024-09-21T10:24:53.267 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:53.305 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:53.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:53.319 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:24:53.318670+0000 min lat: 0.0011678 max lat: 1.0471 avg lat: 0.00719365 2024-09-21T10:24:53.320 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:24:53.320 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 20 16 44479 44463 138.919 25.5 0.0041218 0.00719365 2024-09-21T10:24:53.406 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:53.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.441 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:53.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:53 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:53 smithi045 ceph-mon[95976]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:24:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:53 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3622417417' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:24:53.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:53 smithi045 ceph-mon[95976]: osdmap e383: 8 total, 7 up, 8 in 2024-09-21T10:24:53.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:53.578 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:53.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:53.674 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:24:53 smithi032 podman[267112]: 2024-09-21 10:24:53.344694605 +0000 UTC m=+2.404581416 container cleanup 0e29e15aee7335c2b9bd21eae3f5e8bce7549f6cec4437eef84ee18f06e2e718 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T10:24:53.674 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:24:53 smithi032 bash[267112]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:24:53.680 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:53.717 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:53.819 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:53.858 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:53.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:53.960 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:53.997 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:53.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:54.099 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:54.134 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:54.236 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:54.271 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:54.302 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:54 smithi032 ceph-mon[105748]: pgmap v1153: 99 pgs: 14 stale+active+clean, 85 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-21T10:24:54.302 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:54 smithi032 ceph-mon[105748]: osdmap e384: 8 total, 7 up, 8 in 2024-09-21T10:24:54.302 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:54 smithi032 ceph-mon[101299]: pgmap v1153: 99 pgs: 14 stale+active+clean, 85 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-21T10:24:54.302 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:54 smithi032 ceph-mon[101299]: osdmap e384: 8 total, 7 up, 8 in 2024-09-21T10:24:54.319 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 21 16 46516 46500 138.365 127.312 0.00897927 0.00722214 2024-09-21T10:24:54.373 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:54.399 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:54.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:54.501 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:54 smithi045 ceph-mon[95976]: pgmap v1153: 99 pgs: 14 stale+active+clean, 85 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-21T10:24:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:54 smithi045 ceph-mon[95976]: osdmap e384: 8 total, 7 up, 8 in 2024-09-21T10:24:54.539 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:54.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:54.616 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:24:54 smithi032 podman[267336]: 2024-09-21 10:24:54.396473311 +0000 UTC m=+1.667522603 container remove 0e29e15aee7335c2b9bd21eae3f5e8bce7549f6cec4437eef84ee18f06e2e718 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True) 2024-09-21T10:24:54.642 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:54.652 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:24:54.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.681 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:54.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:54.686 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:24:54.687 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:24:54.717 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:24:54.717 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:24:54.750 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:24:54.750 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:24:54.781 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:24:54.781 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:24:54.782 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:54.816 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:54.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:54.819 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:24:54.819 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:24:54.856 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:24:54.856 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:24:54.889 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:24:54.889 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:24:54.918 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:54.956 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:55.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:55.094 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:55.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2298306263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2298306263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:55.236 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:55.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:55.320 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 22 16 48397 48381 137.419 117.562 0.0137937 0.00727231 2024-09-21T10:24:55.337 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:55.376 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:55.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:55.478 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:55.515 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:55.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:55.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2298306263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:55.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:55.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:55.757 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:55.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.799 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:55.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:55.900 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:55.937 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:55.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:56.039 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:56.059 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:24:56.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:56.179 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:56.216 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:56.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:56.406 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 23 15 50304 50289 136.622 119.25 0.00981315 0.00731411 2024-09-21T10:24:56.406 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:56 smithi032 ceph-mon[101299]: pgmap v1155: 99 pgs: 5 active+undersized, 13 stale+active+clean, 3 active+undersized+degraded, 78 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 2852/127635 objects degraded (2.234%); 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:24:56.406 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:56 smithi032 ceph-mon[101299]: Health check failed: Degraded data redundancy: 2852/127635 objects degraded (2.234%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:56.406 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:56 smithi032 ceph-mon[105748]: pgmap v1155: 99 pgs: 5 active+undersized, 13 stale+active+clean, 3 active+undersized+degraded, 78 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 2852/127635 objects degraded (2.234%); 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:24:56.406 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:56 smithi032 ceph-mon[105748]: Health check failed: Degraded data redundancy: 2852/127635 objects degraded (2.234%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:24:56.432 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:24:56.434 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:56.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:56 smithi045 ceph-mon[95976]: pgmap v1155: 99 pgs: 5 active+undersized, 13 stale+active+clean, 3 active+undersized+degraded, 78 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 2852/127635 objects degraded (2.234%); 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:24:56.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:56 smithi045 ceph-mon[95976]: Health check failed: Degraded data redundancy: 2852/127635 objects degraded (2.234%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:56.536 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:56.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:56.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:56.677 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:56.712 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:56.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:56.814 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.853 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:56.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:56.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:56.993 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:57.095 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:57.125 INFO:teuthology.orchestra.run.smithi032.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2024-09-21T10:24:57.129 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:57.231 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:57.267 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:57.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:57.321 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 24 16 52086 52070 135.567 111.312 0.0110519 0.0073712 2024-09-21T10:24:57.369 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.410 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:57.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:57.511 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:57.547 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:57.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:57.649 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:57.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:57.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:57.763 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:57.790 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.826 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:57.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:57.927 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:57.966 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:57.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:58.068 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:58.106 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:58.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:24:58 smithi032 ceph-mon[105748]: pgmap v1156: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 11711/142641 objects degraded (8.210%); 16 KiB/s, 0 objects/s recovering 2024-09-21T10:24:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:24:58 smithi032 ceph-mon[101299]: pgmap v1156: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 11711/142641 objects degraded (8.210%); 16 KiB/s, 0 objects/s recovering 2024-09-21T10:24:58.208 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:58.242 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:58.321 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 25 16 53922 53906 134.734 114.75 0.014268 0.00741674 2024-09-21T10:24:58.344 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:58.385 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:58.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:58.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:24:58.486 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:58.522 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:58.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:58.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:24:58 smithi045 ceph-mon[95976]: pgmap v1156: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 11711/142641 objects degraded (8.210%); 16 KiB/s, 0 objects/s recovering 2024-09-21T10:24:58.624 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:58.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:58.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:58.760 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:58.797 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:58.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:58.898 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:58.933 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:24:58.976 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:24:59.035 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:59.070 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:59.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.218 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:59.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:59.319 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:59.321 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 26 16 55692 55676 133.807 110.625 0.00807518 0.00746857 2024-09-21T10:24:59.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.361 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:24:59.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:59.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:59.521 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:59.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:59.624 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:59.658 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:59.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:59.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:59.795 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:59.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:59.897 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:59.932 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:24:59.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:59.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:24:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:24:59] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.51.0" 2024-09-21T10:24:59.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:24:59 smithi032 podman[267874]: 2024-09-21 10:24:59.495840794 +0000 UTC m=+0.047418436 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:25:00.034 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:00.068 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:00.070 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:00.084 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:00.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:00.220 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:00.321 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:00.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 27 16 57418 57402 132.846 107.875 0.00723282 0.00752241 2024-09-21T10:25:00.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:00 smithi032 ceph-mon[101299]: pgmap v1157: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 11711/142641 objects degraded (8.210%); 16 KiB/s, 0 objects/s recovering 2024-09-21T10:25:00.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:00 smithi032 ceph-mon[105748]: pgmap v1157: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 11711/142641 objects degraded (8.210%); 16 KiB/s, 0 objects/s recovering 2024-09-21T10:25:00.333 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:00 smithi032 podman[267874]: 2024-09-21 10:25:00.183799193 +0000 UTC m=+0.735376827 container create 7cf5ed6cff4bc9ebe519188c3e190d8af78bcfe3105044de014724cd2f81ecc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T10:25:00.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:00.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:00.457 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:00.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.502 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:00.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:00.531 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:00 smithi045 ceph-mon[95976]: pgmap v1157: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 11711/142641 objects degraded (8.210%); 16 KiB/s, 0 objects/s recovering 2024-09-21T10:25:00.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:00.643 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:00.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:00.656 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:00 smithi032 podman[267874]: 2024-09-21 10:25:00.488079007 +0000 UTC m=+1.039656635 container init 7cf5ed6cff4bc9ebe519188c3e190d8af78bcfe3105044de014724cd2f81ecc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-21T10:25:00.656 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:00 smithi032 podman[267874]: 2024-09-21 10:25:00.494332814 +0000 UTC m=+1.045910441 container start 7cf5ed6cff4bc9ebe519188c3e190d8af78bcfe3105044de014724cd2f81ecc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T10:25:00.745 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:00.782 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:00.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:00.883 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:00.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:00.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:01.022 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:01.056 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:01.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:01.158 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:01.193 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:01.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:01.295 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:01.321 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 28 16 59064 59048 131.775 102.875 0.00670068 0.00758385 2024-09-21T10:25:01.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:01.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:01.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:01.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:01 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14047/166476 objects degraded (8.438%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:01.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:00 smithi032 podman[267874]: 2024-09-21 10:25:00.965380475 +0000 UTC m=+1.516958114 container attach 7cf5ed6cff4bc9ebe519188c3e190d8af78bcfe3105044de014724cd2f81ecc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T10:25:01.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:01 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14047/166476 objects degraded (8.438%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:01.471 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:01.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:01.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:01 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14047/166476 objects degraded (8.438%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:01.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:01.614 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:01.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:01.716 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:01.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:01.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:01.855 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:01.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:01.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:02.034 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:02.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:02.136 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:02.171 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:02.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:02.273 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:02.315 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:02.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:02.321 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 29 16 60751 60735 130.867 105.438 0.0186916 0.00763607 2024-09-21T10:25:02.417 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:02.439 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:02 smithi032 ceph-mon[101299]: pgmap v1158: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 14047/166476 objects degraded (8.438%); 15 KiB/s, 0 objects/s recovering 2024-09-21T10:25:02.439 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:02 smithi032 ceph-mon[105748]: pgmap v1158: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 14047/166476 objects degraded (8.438%); 15 KiB/s, 0 objects/s recovering 2024-09-21T10:25:02.453 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:02.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:02.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:02 smithi045 ceph-mon[95976]: pgmap v1158: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 14047/166476 objects degraded (8.438%); 15 KiB/s, 0 objects/s recovering 2024-09-21T10:25:02.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:02.591 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:02.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:02.693 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:02.727 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:02.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:02.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:02.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.874 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:02.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:02.975 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:03.020 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:03.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:03.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:03.157 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:03.259 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:03.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.288 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:03.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:03.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 30 16 62313 62297 129.759 97.625 0.00396724 0.00770069 2024-09-21T10:25:03.389 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:03.426 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:03.528 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:03.562 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:03.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:03.664 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:03.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.701 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:03.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:03.802 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:03.840 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:03.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:03.942 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:03.986 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:03.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:04.088 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:04.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:04.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:04 smithi032 ceph-mon[101299]: pgmap v1159: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 14047/169875 objects degraded (8.269%); 13 KiB/s, 0 objects/s recovering 2024-09-21T10:25:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:04 smithi032 ceph-mon[105748]: pgmap v1159: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 14047/169875 objects degraded (8.269%); 13 KiB/s, 0 objects/s recovering 2024-09-21T10:25:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:04.230 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:04.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:04.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:04.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 31 16 63858 63842 128.688 96.5625 0.0111946 0.00776636 2024-09-21T10:25:04.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:04.403 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:04.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:04.505 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:04 smithi045 ceph-mon[95976]: pgmap v1159: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 14047/169875 objects degraded (8.269%); 13 KiB/s, 0 objects/s recovering 2024-09-21T10:25:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:04.542 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:04.645 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:04.677 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:04.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:04.779 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:04.822 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:04.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:04.924 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:04.955 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:25:04.962 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:04.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:04.987 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:25:04.987 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:25:05.027 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:25:05.027 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:25:05.064 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:05.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:25:05.069 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:25:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.104 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:05.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:05.105 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:25:05.105 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:25:05.143 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:25:05.143 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:25:05.177 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:25:05.178 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:25:05.205 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:05.220 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:25:05.220 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:25:05.254 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:05.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:05.272 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3047427659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:05.272 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3047427659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:05.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 32 16 65408 65392 127.693 96.875 0.00909893 0.00782614 2024-09-21T10:25:05.356 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:05.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.397 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:05.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:05.497 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.524 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:05.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:05.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3047427659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:05.624 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:05.659 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:05.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:05.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:05.792 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:05.894 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:05.932 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:05.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:06.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:06.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:06.174 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:06.205 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:25:06.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:06.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:06.228 INFO:teuthology.orchestra.run.smithi032.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2024-09-21T10:25:06.311 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:06.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 33 16 66917 66901 126.682 94.3125 0.0140556 0.00788906 2024-09-21T10:25:06.344 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:06.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:06.446 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:06.450 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:06 smithi032 ceph-mon[101299]: pgmap v1160: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 14555/176175 objects degraded (8.262%); 11 KiB/s, 0 objects/s recovering 2024-09-21T10:25:06.450 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3276472899' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:25:06.451 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:06 smithi032 ceph-mon[105748]: pgmap v1160: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 14555/176175 objects degraded (8.262%); 11 KiB/s, 0 objects/s recovering 2024-09-21T10:25:06.451 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3276472899' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:25:06.451 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:06 smithi032 podman[267874]: 2024-09-21 10:25:06.223037329 +0000 UTC m=+6.774614970 container died 7cf5ed6cff4bc9ebe519188c3e190d8af78bcfe3105044de014724cd2f81ecc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T10:25:06.478 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:06.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:06.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:06 smithi045 ceph-mon[95976]: pgmap v1160: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 14555/176175 objects degraded (8.262%); 11 KiB/s, 0 objects/s recovering 2024-09-21T10:25:06.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3276472899' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:25:06.581 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:06.617 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:06.719 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:06.758 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:06.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:06.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:06.996 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:07.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:07.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:07.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:07.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:07.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:07.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:07.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:07.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:07.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 34 16 68463 68447 125.797 96.625 0.0125814 0.00794442 2024-09-21T10:25:07.413 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:07.450 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:07 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:25:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:07 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3276472899' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:25:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:07 smithi045 ceph-mon[95976]: osdmap e385: 8 total, 7 up, 8 in 2024-09-21T10:25:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:07 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14555/176175 objects degraded (8.262%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:07.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.584 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:07.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:07.685 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:07.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:07 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:25:07.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:07 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3276472899' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:25:07.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:07 smithi032 ceph-mon[105748]: osdmap e385: 8 total, 7 up, 8 in 2024-09-21T10:25:07.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:07 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14555/176175 objects degraded (8.262%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:07.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:07 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:25:07.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:07 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3276472899' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:25:07.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:07 smithi032 ceph-mon[101299]: osdmap e385: 8 total, 7 up, 8 in 2024-09-21T10:25:07.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:07 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14555/176175 objects degraded (8.262%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:07.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.723 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:07.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:07.824 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:07.861 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:07.963 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:07.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.989 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:07.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:08.090 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:08.130 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:08.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:08.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:08.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:08.270 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:08.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:08.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 35 16 70034 70018 125.008 98.1875 0.0108822 0.00799443 2024-09-21T10:25:08.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:08.394 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:08.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:08.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:08 smithi045 ceph-mon[95976]: pgmap v1162: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 16135/194256 objects degraded (8.306%) 2024-09-21T10:25:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:08 smithi045 ceph-mon[95976]: 1.a scrub starts 2024-09-21T10:25:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:08 smithi045 ceph-mon[95976]: 1.a scrub ok 2024-09-21T10:25:08.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:08.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:08.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:08.558 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:08 smithi032 ceph-mon[101299]: pgmap v1162: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 16135/194256 objects degraded (8.306%) 2024-09-21T10:25:08.558 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:08 smithi032 ceph-mon[101299]: 1.a scrub starts 2024-09-21T10:25:08.558 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:08 smithi032 ceph-mon[101299]: 1.a scrub ok 2024-09-21T10:25:08.558 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:08 smithi032 ceph-mon[105748]: pgmap v1162: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 16135/194256 objects degraded (8.306%) 2024-09-21T10:25:08.558 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:08 smithi032 ceph-mon[105748]: 1.a scrub starts 2024-09-21T10:25:08.559 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:08 smithi032 ceph-mon[105748]: 1.a scrub ok 2024-09-21T10:25:08.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:08.679 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:08.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:08.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:08.781 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:08.816 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:08.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:08.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:08.918 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:08.955 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:08.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:08.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:09.057 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:09.092 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:09.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:09.193 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:09.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:09.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:09.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:09.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 36 16 71479 71463 124.045 90.3125 0.0112843 0.00805687 2024-09-21T10:25:09.331 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:09.363 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:09.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:09.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:09.465 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:09.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:09.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:09.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:09.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:09.623 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:09.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:09.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:09.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:09.761 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:09.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:09.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:09.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:09.898 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:09.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:09.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:25:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:25:09] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.51.0" 2024-09-21T10:25:10.000 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:10.037 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:10.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:10.139 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:10.172 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:10.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:10.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:10.274 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:10.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:10.315 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:10.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:10.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 37 16 72993 72977 123.249 94.625 0.00643383 0.00810918 2024-09-21T10:25:10.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:10.452 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:10.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:10 smithi045 ceph-mon[95976]: pgmap v1163: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 16135/194256 objects degraded (8.306%) 2024-09-21T10:25:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:10 smithi045 ceph-mon[95976]: 1.8 scrub starts 2024-09-21T10:25:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:10 smithi045 ceph-mon[95976]: 1.8 scrub ok 2024-09-21T10:25:10.554 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:10.577 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:10.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:10.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:10.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:10 smithi032 ceph-mon[101299]: pgmap v1163: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 16135/194256 objects degraded (8.306%) 2024-09-21T10:25:10.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:10 smithi032 ceph-mon[101299]: 1.8 scrub starts 2024-09-21T10:25:10.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:10 smithi032 ceph-mon[101299]: 1.8 scrub ok 2024-09-21T10:25:10.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:10 smithi032 ceph-mon[105748]: pgmap v1163: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 16135/194256 objects degraded (8.306%) 2024-09-21T10:25:10.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:10 smithi032 ceph-mon[105748]: 1.8 scrub starts 2024-09-21T10:25:10.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:10 smithi032 ceph-mon[105748]: 1.8 scrub ok 2024-09-21T10:25:10.713 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:10.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:10.815 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:10.851 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:10.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:10.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:10.953 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:10.985 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:10.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:10.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:11.087 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:11.118 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:11.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:11.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:11.220 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:11.258 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:11.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:11.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:11.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 38 16 74440 74424 122.386 90.4375 0.0108182 0.0081663 2024-09-21T10:25:11.360 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:11.397 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:11.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:11.498 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:11.533 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:11.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:11.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:11.635 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:11.670 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:11.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:11.771 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:11.808 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:11.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:11.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:11.910 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:11.947 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:11.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:11.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:12.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:12.085 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:12.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:12.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:12.188 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:12.233 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:12.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:12.322 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 39 16 75896 75880 121.581 91 0.0102651 0.00822041 2024-09-21T10:25:12.335 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:12.369 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:12.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:12.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:12 smithi032 ceph-mon[101299]: pgmap v1164: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 17948/214248 objects degraded (8.377%) 2024-09-21T10:25:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:12 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 17948/214248 objects degraded (8.377%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:12.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:12 smithi032 ceph-mon[105748]: pgmap v1164: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 17948/214248 objects degraded (8.377%) 2024-09-21T10:25:12.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:12 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 17948/214248 objects degraded (8.377%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:12.471 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:12.508 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:12.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:12.527 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T10:25:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:12 smithi045 ceph-mon[95976]: pgmap v1164: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 17948/214248 objects degraded (8.377%) 2024-09-21T10:25:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:12 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 17948/214248 objects degraded (8.377%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:12.610 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:12.650 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:12.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:12.679 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:25:12.679 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T10:25:12.688 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:25:12.688 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:25:12.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:12.807 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:12.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:25:12.835 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:25:12.909 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:12.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:12.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:12.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:13.049 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:13.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:13.090 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:13.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:13.191 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:13.238 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:13.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:13.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:25:13.321686+0000 min lat: 0.0011678 max lat: 1.0471 avg lat: 0.00827162 2024-09-21T10:25:13.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:25:13.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 40 16 77356 77340 120.822 91.25 0.00982312 0.00827162 2024-09-21T10:25:13.338 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:13.375 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:13.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:13.477 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:13.514 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:13.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:13.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:13.616 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:13.657 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:13.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:13.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:13.759 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:13.800 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:13.801 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:13.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:13.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:13.903 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:13.942 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:13.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:14.044 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:14.079 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:14.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:14.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:14.181 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:14.220 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:14.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:14.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:14.322 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:14.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 41 16 78787 78771 120.057 89.4375 0.00737801 0.00832429 2024-09-21T10:25:14.365 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:14.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:14 smithi032 ceph-mon[101299]: pgmap v1165: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 17948/217029 objects degraded (8.270%) 2024-09-21T10:25:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:14 smithi032 ceph-mon[105748]: pgmap v1165: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 17948/217029 objects degraded (8.270%) 2024-09-21T10:25:14.466 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:14.505 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:14.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:14 smithi045 ceph-mon[95976]: pgmap v1165: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 17948/217029 objects degraded (8.270%) 2024-09-21T10:25:14.606 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:14.637 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:14.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:14.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:14.739 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:14.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:14.774 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:14.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:14.835 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:25:14.875 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:14.922 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:14.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:14.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:15.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:15.060 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:15.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:15.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:15.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:15.200 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:15.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:15.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:15.302 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:15.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 42 16 80181 80165 119.272 87.125 0.00772169 0.00837809 2024-09-21T10:25:15.340 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:15.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:15.435 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:15.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:15.478 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:15.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:15.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:15.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:15.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:15.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:15.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:15.664 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:25:15.709 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:15.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:15.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:15.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:15.891 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:15.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:15.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:15.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:16.029 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:16.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:16.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:16.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:16.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:16.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:16.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:16.262 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:16.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:16.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:16.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:16.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 43 16 81512 81496 118.433 83.1875 0.00771444 0.008439 2024-09-21T10:25:16.406 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:16.449 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:16.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:16.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:16.499 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:25:16.499 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:25:16.500 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:16 smithi032 ceph-mon[101299]: pgmap v1166: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 18439/222717 objects degraded (8.279%) 2024-09-21T10:25:16.500 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:16 smithi032 ceph-mon[105748]: pgmap v1166: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 18439/222717 objects degraded (8.279%) 2024-09-21T10:25:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:16 smithi045 ceph-mon[95976]: pgmap v1166: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 18439/222717 objects degraded (8.279%) 2024-09-21T10:25:16.551 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:16.587 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:16.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:16.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:16.688 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:16.733 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:16.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:16.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:16.835 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:16.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:16.930 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:16.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:17.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:17.064 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:17.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:17.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:17.166 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:17.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:17.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:17.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:17.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:17.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:25:17.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 44 16 82900 82884 117.712 86.75 0.00864378 0.00849011 2024-09-21T10:25:17.348 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:17.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:17.449 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:17.496 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:17.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:17 smithi045 ceph-mon[95976]: pgmap v1167: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 19838/238806 objects degraded (8.307%) 2024-09-21T10:25:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 18439/222717 objects degraded (8.279%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:17.598 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:17.636 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:17.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:17.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:17.679 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 1, 0, 4] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 6, 7, 5] 2024-09-21T10:25:17.680 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:25:17.680 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2024-09-21T10:25:17.680 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2024-09-21T10:25:17.680 DEBUG:teuthology.orchestra.run.smithi032:osd.0> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T10:25:17.683 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl start ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:25:17.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:17 smithi032 ceph-mon[101299]: pgmap v1167: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 19838/238806 objects degraded (8.307%) 2024-09-21T10:25:17.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 18439/222717 objects degraded (8.279%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:17.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:17 smithi032 ceph-mon[105748]: pgmap v1167: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 19838/238806 objects degraded (8.307%) 2024-09-21T10:25:17.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 18439/222717 objects degraded (8.279%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:17.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:17.783 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:17.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:17.884 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:17.912 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:17.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:17.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:17.995 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:18.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:18.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:18.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:18.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:18.073 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:17 smithi032 systemd[1]: Starting Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:25:18.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:18.189 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:18.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:18.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:18.291 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:18.317 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:18.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:18.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 45 16 84177 84161 116.87 79.8125 0.0167741 0.00854989 2024-09-21T10:25:18.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:18.453 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:18.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:18.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:18.555 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:18.593 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:18.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:18.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:18.635 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:18.636 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:18.646 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:18.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:18.696 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:25:18.736 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:25:18.736 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:25:18.744 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:18.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:18.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:18.771 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:25:18.771 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:25:18.803 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:25:18.803 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:25:18.837 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:25:18.837 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:25:18.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:18.876 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:25:18.877 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:25:18.886 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:18.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:18.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:18.905 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:25:18.906 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:25:18.906 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:18 smithi032 podman[269665]: 2024-09-21 10:25:18.712322076 +0000 UTC m=+0.476248334 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:25:18.945 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:25:18.946 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:25:18.987 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:19.024 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:19.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:19.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:19.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:19.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:19.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:19.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:19.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:19.271 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:19.309 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:19.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:19.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 46 16 85427 85411 116.028 78.125 0.0214299 0.0086132 2024-09-21T10:25:19.411 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:19.446 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:19.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:19.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:19.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:19.583 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:19.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:19.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:19.685 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:19.727 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:19.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:19 smithi045 ceph-mon[95976]: pgmap v1168: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 19838/238806 objects degraded (8.307%) 2024-09-21T10:25:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/466572352' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:19.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:19.865 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:19.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:19.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:19.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:25:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:25:19] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.51.0" 2024-09-21T10:25:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:19 smithi032 ceph-mon[101299]: pgmap v1168: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 19838/238806 objects degraded (8.307%) 2024-09-21T10:25:19.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/466572352' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:19 smithi032 ceph-mon[105748]: pgmap v1168: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 19838/238806 objects degraded (8.307%) 2024-09-21T10:25:19.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/466572352' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:19.969 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:20.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:20.013 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:20.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:20.114 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:20.154 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:20.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:20.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:20.256 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:20.293 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:20.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:20.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:20.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 47 16 86494 86478 114.978 66.6875 0.0168901 0.00869242 2024-09-21T10:25:20.394 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:20.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:20.427 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:20.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:20.528 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:20.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:20.571 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:20.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:20.672 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:20.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:20.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:20.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:20.810 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:20.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:20.845 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:20.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:20.946 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:20.980 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:20.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:20.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:21.082 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:21.116 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:21.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:21.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:21.126 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:20 smithi032 podman[269665]: 2024-09-21 10:25:20.855085747 +0000 UTC m=+2.619011997 container create c620244e195e5dceeff62ecac1b9af886adb3d7b67f21c88027f2e05dd16dbd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T10:25:21.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:21.255 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:21.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:21.324 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 48 16 87698 87682 114.15 75.25 0.0173981 0.00875548 2024-09-21T10:25:21.357 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:21.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:21.401 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:21.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:21.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:21.544 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:21.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:21.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:21.646 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:21.686 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:21.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:21.788 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:21.826 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:21.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:21.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:21.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:21 smithi032 ceph-mon[101299]: pgmap v1169: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 107 MiB/s wr, 1.70k op/s; 21442/255693 objects degraded (8.386%) 2024-09-21T10:25:21.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:21 smithi032 ceph-mon[105748]: pgmap v1169: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 107 MiB/s wr, 1.70k op/s; 21442/255693 objects degraded (8.386%) 2024-09-21T10:25:21.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:21.968 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:21.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:22.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:21 smithi045 ceph-mon[95976]: pgmap v1169: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 107 MiB/s wr, 1.70k op/s; 21442/255693 objects degraded (8.386%) 2024-09-21T10:25:22.069 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:22.105 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:22.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:22.207 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:22.246 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:22.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:22.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:22.323 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 49 16 89008 88992 113.491 81.875 0.012173 0.00880669 2024-09-21T10:25:22.348 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:22.385 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:22.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:22.486 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:22.527 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:22.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:22.535 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:25:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=plugins.update.checker t=2024-09-21T10:25:22.258831806Z level=info msg="Update check succeeded" duration=60.507216ms 2024-09-21T10:25:22.584 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:25:22.585 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:25:22.628 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:22.663 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:22.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:22.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:22.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:22.804 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:22.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:22.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 21442/255693 objects degraded (8.386%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:22.863 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 21442/255693 objects degraded (8.386%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:22.863 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:22 smithi032 podman[269665]: 2024-09-21 10:25:22.680456766 +0000 UTC m=+4.444383020 container init c620244e195e5dceeff62ecac1b9af886adb3d7b67f21c88027f2e05dd16dbd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T10:25:22.863 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:22 smithi032 podman[269665]: 2024-09-21 10:25:22.687113063 +0000 UTC m=+4.451039314 container start c620244e195e5dceeff62ecac1b9af886adb3d7b67f21c88027f2e05dd16dbd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:25:22.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:22.944 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:22.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:22.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:23.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:25:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=cleanup t=2024-09-21T10:25:22.572900987Z level=info msg="Completed cleanup jobs" duration=449.190688ms 2024-09-21T10:25:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 21442/255693 objects degraded (8.386%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:23.046 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:23.085 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:23.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:23.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:23.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:23.232 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:23.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:23.324 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 50 16 90252 90236 112.776 77.75 0.0119804 0.00886236 2024-09-21T10:25:23.333 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:23.369 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:23.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:23.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:23.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:23 smithi032 podman[269665]: 2024-09-21 10:25:23.34985375 +0000 UTC m=+5.113779997 container attach c620244e195e5dceeff62ecac1b9af886adb3d7b67f21c88027f2e05dd16dbd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:25:23.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:23.509 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:23.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:23.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:23.610 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:23.644 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:23.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:23.734 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:23 smithi032 ceph-mon[105748]: pgmap v1170: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 21442/258033 objects degraded (8.310%) 2024-09-21T10:25:23.734 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:23 smithi032 ceph-mon[101299]: pgmap v1170: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 21442/258033 objects degraded (8.310%) 2024-09-21T10:25:23.745 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:23.782 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:23.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:23.883 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:23.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:23.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:23.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:24.018 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:23 smithi045 ceph-mon[95976]: pgmap v1170: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 21442/258033 objects degraded (8.310%) 2024-09-21T10:25:24.060 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:24.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:24.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:24.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:24.196 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:24.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:24.297 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:24.324 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 51 16 91521 91505 112.12 79.3125 0.0137677 0.00891366 2024-09-21T10:25:24.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:24.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:24.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:24.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:24.474 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:24.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:24.544 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[269898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:25:24.544 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:24 smithi032 bash[269665]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:25:24.576 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:24.586 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:25:24.592 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:25:24.592 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:25:24.592 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:25:24.592 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:25:24.592 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:25:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:24.610 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:24.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:24.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:24.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:24.758 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:24.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:24.859 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:24.893 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:24.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:24.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:24.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:24 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[269898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:25:24.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:24 smithi032 bash[269665]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:25:24.995 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:25.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:25.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:25.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:25.133 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:25.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:25.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:25.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:25.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:25.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:25.314 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:25.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:25.324 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 52 16 92773 92757 111.469 78.25 0.0152858 0.0089663 2024-09-21T10:25:25.416 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:25.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:25.452 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:25.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:25.553 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:25.598 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:25.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:25.699 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:25.736 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:25.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:25.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:25.838 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:25.867 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:25.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:25.886 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:25 smithi032 ceph-mon[105748]: pgmap v1171: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 21827/262644 objects degraded (8.310%) 2024-09-21T10:25:25.887 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:25 smithi032 ceph-mon[101299]: pgmap v1171: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 21827/262644 objects degraded (8.310%) 2024-09-21T10:25:25.968 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:26.004 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:26.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:26.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:25 smithi045 ceph-mon[95976]: pgmap v1171: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 21827/262644 objects degraded (8.310%) 2024-09-21T10:25:26.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:26.140 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:26.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:26.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:25 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[269898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:25:26.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:25 smithi032 bash[269665]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:25:26.242 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:26.280 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:26.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:26.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:26.324 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 53 16 93894 93878 110.688 70.0625 0.00541702 0.00902947 2024-09-21T10:25:26.383 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:26.406 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:26.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:26.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:26.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:26.547 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:26.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:26.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:26.586 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[269898]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:25:26.586 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 bash[269665]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:25:26.631 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:25:26.649 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:26.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:26.691 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:26.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:26.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:26.829 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:26.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:26.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:26.931 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:26.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2438291431' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:26.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:26.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2438291431' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:26.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:26.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[269898]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:25:26.958 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 bash[269665]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:25:26.958 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[269898]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:25:26.958 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 bash[269665]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:25:26.958 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[269898]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:25:26.958 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 bash[269665]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:25:26.958 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[269898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:25:26.958 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 bash[269665]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:25:26.958 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[269898]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T10:25:26.958 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:26 smithi032 bash[269665]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T10:25:26.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:26.968 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:26.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:27.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2438291431' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:27.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:26 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:27.069 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:27.111 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:27.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:27.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:27.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:27.247 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:27.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:27.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:27.324 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 54 16 95104 95088 110.038 75.625 0.00437295 0.00908268 2024-09-21T10:25:27.349 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:27.383 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:27.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:27.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:27.485 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:27.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:27.520 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:27.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:27.621 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:27.654 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:27.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:27.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:27.778 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:27.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:27.879 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:27.921 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:27.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:27.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[105748]: pgmap v1172: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 23056/276723 objects degraded (8.332%) 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[105748]: osdmap e386: 8 total, 7 up, 8 in 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 23056/276723 objects degraded (8.332%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[101299]: pgmap v1172: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 23056/276723 objects degraded (8.332%) 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[101299]: osdmap e386: 8 total, 7 up, 8 in 2024-09-21T10:25:27.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:27 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 23056/276723 objects degraded (8.332%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:28.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:27 smithi045 ceph-mon[95976]: pgmap v1172: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 23056/276723 objects degraded (8.332%) 2024-09-21T10:25:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:27 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:25:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:27 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:27 smithi045 ceph-mon[95976]: osdmap e386: 8 total, 7 up, 8 in 2024-09-21T10:25:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:27 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 23056/276723 objects degraded (8.332%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:28.062 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:28.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:28.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:28.164 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:28.196 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:28.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:28.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:28.298 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:28.324 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 55 16 96242 96226 109.331 71.125 0.0135301 0.00914178 2024-09-21T10:25:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:28.333 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:28.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:28.384 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:28 smithi032 podman[269665]: 2024-09-21 10:25:28.121863386 +0000 UTC m=+9.885789647 container died c620244e195e5dceeff62ecac1b9af886adb3d7b67f21c88027f2e05dd16dbd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:25:28.393 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:28.434 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:28.474 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:28.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:28.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:28.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:28.618 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:28.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:28.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:28.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:28.759 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:28.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:28.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:28.899 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:28.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:29.000 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:29.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:29.040 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:29.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:29.141 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:29.180 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:29.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:29.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:29.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:28 smithi032 podman[270418]: 2024-09-21 10:25:28.926250508 +0000 UTC m=+2.122166658 container remove c620244e195e5dceeff62ecac1b9af886adb3d7b67f21c88027f2e05dd16dbd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T10:25:29.282 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:29.323 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:29.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:29.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:29.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 56 16 97486 97470 108.767 77.75 0.0104809 0.00918922 2024-09-21T10:25:29.425 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:29.463 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:29.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:29.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:29.565 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:29.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:29.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:29.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:29.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:29.724 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:25:29.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:29.746 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:29.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:29.796 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:29 smithi032 ceph-mon[105748]: pgmap v1174: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 23056/276723 objects degraded (8.332%) 2024-09-21T10:25:29.796 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:25:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:25:29] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.51.0" 2024-09-21T10:25:29.797 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:29 smithi032 ceph-mon[101299]: pgmap v1174: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 23056/276723 objects degraded (8.332%) 2024-09-21T10:25:29.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:29.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:29.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:29.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:30.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:30.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:30.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:29 smithi045 ceph-mon[95976]: pgmap v1174: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 23056/276723 objects degraded (8.332%) 2024-09-21T10:25:30.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:30.177 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:30.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:30.279 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:30.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:30.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:30.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:30.324 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 57 16 98680 98664 108.167 74.625 0.0135374 0.00924001 2024-09-21T10:25:30.422 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:30.461 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:30.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:30.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:30.562 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:30.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:30.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:30.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:30.684 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:30.706 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:30.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:30.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:30.841 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:30.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:30.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:30.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:30.982 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:31.018 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:31.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:31.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:31.120 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:31.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:31.163 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:31.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:31.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:31.311 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:31.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:31.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:31.324 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 58 16 99846 99830 107.559 72.875 0.015809 0.00929248 2024-09-21T10:25:31.414 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:31.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:31.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:31.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:31.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:31.603 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:31.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:31.704 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:31.743 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:31.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:31.847 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:31.883 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:31.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:31 smithi032 ceph-mon[105748]: pgmap v1175: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 24535/292362 objects degraded (8.392%) 2024-09-21T10:25:31.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:31 smithi032 ceph-mon[101299]: pgmap v1175: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 24535/292362 objects degraded (8.392%) 2024-09-21T10:25:31.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:32.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:32.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:32.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:31 smithi045 ceph-mon[95976]: pgmap v1175: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 24535/292362 objects degraded (8.392%) 2024-09-21T10:25:32.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:32.160 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:32.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:32.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:32.261 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:32.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:32.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:32.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 59 16 100963 100947 106.919 69.8125 0.00623942 0.00934864 2024-09-21T10:25:32.384 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:32.417 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:32.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:32.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:32.519 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:32.552 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:32.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:32.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:32.623 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:25:32.654 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:32.712 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:32.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:32.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:32.861 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:32.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:32.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:32.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:32 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 24535/292362 objects degraded (8.392%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:32 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 24535/292362 objects degraded (8.392%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:32.963 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:32.993 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:32.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:32.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:33.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:32 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 24535/292362 objects degraded (8.392%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:33.094 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:33.131 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:33.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:33.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:33.233 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:33.271 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:33.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:33.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:25:33.323755+0000 min lat: 0.0011678 max lat: 1.0471 avg lat: 0.00939815 2024-09-21T10:25:33.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:25:33.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 60 16 102128 102112 106.35 72.8125 0.00706166 0.00939815 2024-09-21T10:25:33.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:33.405 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:33.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:33.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:33.507 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:33.543 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:33.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:33.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:33.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:33.685 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:33.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:33.787 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:33.819 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:33.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:33.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:33.921 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:33.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:33.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:33.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:33.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:33 smithi032 ceph-mon[101299]: pgmap v1176: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 24535/294633 objects degraded (8.327%) 2024-09-21T10:25:33.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:33.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:33 smithi032 ceph-mon[105748]: pgmap v1176: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 24535/294633 objects degraded (8.327%) 2024-09-21T10:25:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:33 smithi045 ceph-mon[95976]: pgmap v1176: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 24535/294633 objects degraded (8.327%) 2024-09-21T10:25:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:34.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:34.059 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:34.094 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:34.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:34.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:34.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:34.222 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:34.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:34.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:34.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:34.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 61 16 103231 103215 105.737 68.9375 0.0138 0.00945225 2024-09-21T10:25:34.369 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:34.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:34.422 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:34 smithi032 podman[271131]: 2024-09-21 10:25:34.216183417 +0000 UTC m=+0.041164476 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:25:34.471 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:34.507 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:34.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:34.583 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:25:34.607 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:34.623 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:25:34.623 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:25:34.636 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:34.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:34.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:34.671 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:25:34.671 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:25:34.700 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:25:34.700 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:25:34.701 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1223073447' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:34.701 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1223073447' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:34.729 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:25:34.730 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:25:34.737 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:34.764 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:25:34.764 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:25:34.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:34.775 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:34.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:34.790 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:25:34.790 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:25:34.820 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:25:34.821 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:25:34.875 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:34.913 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:34.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:34.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:34.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:34 smithi032 podman[271131]: 2024-09-21 10:25:34.739309655 +0000 UTC m=+0.564290703 container create c09f6fabe76b2589964b26e712dbef9a1a02d300b50866157d1df75f498f6de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b) 2024-09-21T10:25:35.015 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1223073447' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:35.049 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:35.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:35.151 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:35.196 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:35.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:35.298 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:35.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 62 16 104324 104308 105.133 68.3125 0.00621979 0.0095065 2024-09-21T10:25:35.343 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:35.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:35.444 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:35.480 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:35.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:35.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:35.582 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:35.610 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:35 smithi032 podman[271131]: 2024-09-21 10:25:35.44863192 +0000 UTC m=+1.273612971 container init c09f6fabe76b2589964b26e712dbef9a1a02d300b50866157d1df75f498f6de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:25:35.610 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:35 smithi032 podman[271131]: 2024-09-21 10:25:35.455933619 +0000 UTC m=+1.280914667 container start c09f6fabe76b2589964b26e712dbef9a1a02d300b50866157d1df75f498f6de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:25:35.610 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:35 smithi032 ceph-osd[271301]: -- 172.21.15.32:0/1130108502 <== mon.0 v2:172.21.15.32:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56281e6e5a40 con 0x56281f4b4000 2024-09-21T10:25:35.615 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:35.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:35.717 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:35.758 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:35.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:35.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:35.766 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:25:35.766 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:25:35.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:35.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:35.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:35 smithi032 ceph-mon[105748]: pgmap v1177: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 24893/298893 objects degraded (8.328%) 2024-09-21T10:25:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:35 smithi032 ceph-mon[101299]: pgmap v1177: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 24893/298893 objects degraded (8.328%) 2024-09-21T10:25:36.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:35 smithi045 ceph-mon[95976]: pgmap v1177: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 24893/298893 objects degraded (8.328%) 2024-09-21T10:25:36.041 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:36.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:36.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:36.143 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:36.181 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:36.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:36.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:36.282 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:36.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 63 16 105477 105461 104.608 72.0625 0.0154495 0.0095543 2024-09-21T10:25:36.328 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:36.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:36.430 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:36.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:36.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:36.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:36.521 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:25:36.569 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:36.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:25:36.607 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:36.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:36.621 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:36.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:36.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:36 smithi032 bash[271131]: c09f6fabe76b2589964b26e712dbef9a1a02d300b50866157d1df75f498f6de6 2024-09-21T10:25:36.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:36 smithi032 systemd[1]: Started Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:25:36.721 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:36.754 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:36.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:36.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:36.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:36.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:36.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:36.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:36.997 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:37.038 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:37.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:37.088 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:25:37.088 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:25:37.139 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:37.185 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:37.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:37.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:37.287 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:37.476 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 64 15 106586 106571 104.058 69.375 0.00949703 0.00960526 2024-09-21T10:25:37.476 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[271261]: 2024-09-21T10:25:37.206+0000 7ff59ea3f640 -1 Falling back to public interface 2024-09-21T10:25:37.513 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:37.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:37.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:37.615 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:37.650 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:37.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:37.752 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:37.784 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:37.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:37.886 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:37.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:37.930 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:37.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:37.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:37 smithi032 ceph-mon[101299]: pgmap v1178: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 25985/311811 objects degraded (8.334%) 2024-09-21T10:25:37.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:37 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 25985/311811 objects degraded (8.334%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:37.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:37 smithi032 ceph-mon[105748]: pgmap v1178: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 25985/311811 objects degraded (8.334%) 2024-09-21T10:25:37.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:37 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 25985/311811 objects degraded (8.334%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:38.031 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:37 smithi045 ceph-mon[95976]: pgmap v1178: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 25985/311811 objects degraded (8.334%) 2024-09-21T10:25:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:37 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 25985/311811 objects degraded (8.334%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:38.069 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:38.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:38.170 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:38.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:38.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:38.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:38.311 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:38.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 65 16 107722 107706 103.548 70.9375 0.0109773 0.00965216 2024-09-21T10:25:38.345 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:38.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:38.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:38.447 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:38.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:38.483 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:38.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:38.549 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:38.583 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:38.596 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:38.642 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:38.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:38.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:38.784 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:38.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:38.886 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:38.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:38.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:39.017 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:39.053 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:39.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:39.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:39.090 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:25:39.155 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:39.189 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:39.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:39.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:39.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:39.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 66 16 108808 108792 103.008 67.875 0.0112019 0.00970233 2024-09-21T10:25:39.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:39.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:39.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:39.440 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:39.482 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:39.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:39.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:39.583 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:39.626 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:39.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:39.728 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:39.766 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:39.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:39.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:39.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:39 smithi032 ceph-mon[101299]: pgmap v1179: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 25985/311811 objects degraded (8.334%) 2024-09-21T10:25:39.826 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:25:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:25:39] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.51.0" 2024-09-21T10:25:39.826 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:39 smithi032 ceph-mon[105748]: pgmap v1179: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 25985/311811 objects degraded (8.334%) 2024-09-21T10:25:39.867 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:39.908 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:39.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:40.010 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:39 smithi045 ceph-mon[95976]: pgmap v1179: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 25985/311811 objects degraded (8.334%) 2024-09-21T10:25:40.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:40.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:40.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:40.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:40.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:40.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:40.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:40.288 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:40.322 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:40.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:40.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 67 16 109849 109833 102.441 65.0625 0.00871717 0.00975625 2024-09-21T10:25:40.424 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:40.462 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:40.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:40.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:40.564 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:40.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:40.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:40.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:40.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:40.737 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:40.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:40.839 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:40.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:40.872 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:40.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:40.973 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:41.016 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:41.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:41.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:41.117 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:41.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:41.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:41.256 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:41.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:41.294 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:41.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:41.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 68 16 110836 110820 101.842 61.6875 0.0156214 0.00981463 2024-09-21T10:25:41.395 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:41.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:41.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:41.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:41.533 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:41.577 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:41.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:41.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:41.679 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:41.709 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:25:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:41.744 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:41.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:41.845 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:41.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:41.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:41.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:41 smithi032 ceph-mon[101299]: pgmap v1180: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 27386/326271 objects degraded (8.394%) 2024-09-21T10:25:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:41 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3652312883' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:25:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:41 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:25:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:41 smithi032 ceph-mon[105748]: pgmap v1180: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 27386/326271 objects degraded (8.394%) 2024-09-21T10:25:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:41 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3652312883' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:25:41.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:41 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:25:41.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:42.022 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:42.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:42.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:41 smithi045 ceph-mon[95976]: pgmap v1180: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 27386/326271 objects degraded (8.394%) 2024-09-21T10:25:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:41 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3652312883' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:25:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:41 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:25:42.124 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:42.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:42.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:42.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:42.269 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:42.313 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:42.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:42.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:42.325 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 69 16 111772 111756 101.213 58.5 0.0215046 0.00987532 2024-09-21T10:25:42.415 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:42.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:42.450 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:42.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:42.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:42.585 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:42.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:42.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:42.687 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:42.723 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:42.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:42.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:42.826 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:42.886 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:42.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:42.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:42 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:25:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:42 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3652312883' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:25:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:42 smithi032 ceph-mon[101299]: osdmap e387: 8 total, 7 up, 8 in 2024-09-21T10:25:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:42 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 27386/326271 objects degraded (8.394%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:42 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:25:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:42 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3652312883' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:25:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:42 smithi032 ceph-mon[105748]: osdmap e387: 8 total, 7 up, 8 in 2024-09-21T10:25:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:42 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 27386/326271 objects degraded (8.394%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:42.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:43.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:43.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:43.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:42 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:25:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:42 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3652312883' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:25:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:42 smithi045 ceph-mon[95976]: osdmap e387: 8 total, 7 up, 8 in 2024-09-21T10:25:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:42 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 27386/326271 objects degraded (8.394%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:43.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:43.158 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:43.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:43.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:43.259 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:43.295 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:43.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:43.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:43.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 70 16 112812 112796 100.696 65 0.0150859 0.00992661 2024-09-21T10:25:43.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:43.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:43.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:43.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:43.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:43.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:43.595 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:43.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:43.696 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:43.729 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:43.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:43.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:43.830 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:43.864 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:43.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:43.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:43 smithi032 ceph-mon[105748]: pgmap v1182: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 27386/328446 objects degraded (8.338%) 2024-09-21T10:25:43.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:43 smithi032 ceph-mon[101299]: pgmap v1182: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 27386/328446 objects degraded (8.338%) 2024-09-21T10:25:43.966 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:44.001 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:44.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:43 smithi045 ceph-mon[95976]: pgmap v1182: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 27386/328446 objects degraded (8.338%) 2024-09-21T10:25:44.103 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:44.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:44.142 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:44.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:44.243 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:44.290 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:44.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:44.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:44.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 71 16 113752 113736 100.105 58.75 0.00905699 0.00998394 2024-09-21T10:25:44.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:44.434 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:44.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:44.536 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:44.570 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:44.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:44.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:44.672 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:44.728 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:44.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:44.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:44.776 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:44.824 INFO:teuthology.orchestra.run.smithi032.stderr:no valid command found; 10 closest matches: 2024-09-21T10:25:44.824 INFO:teuthology.orchestra.run.smithi032.stderr:0 2024-09-21T10:25:44.824 INFO:teuthology.orchestra.run.smithi032.stderr:1 2024-09-21T10:25:44.824 INFO:teuthology.orchestra.run.smithi032.stderr:2 2024-09-21T10:25:44.825 INFO:teuthology.orchestra.run.smithi032.stderr:abort 2024-09-21T10:25:44.825 INFO:teuthology.orchestra.run.smithi032.stderr:assert 2024-09-21T10:25:44.825 INFO:teuthology.orchestra.run.smithi032.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:25:44.825 INFO:teuthology.orchestra.run.smithi032.stderr:bluefs files list 2024-09-21T10:25:44.825 INFO:teuthology.orchestra.run.smithi032.stderr:bluefs stats 2024-09-21T10:25:44.825 INFO:teuthology.orchestra.run.smithi032.stderr:bluestore allocator dump block 2024-09-21T10:25:44.825 INFO:teuthology.orchestra.run.smithi032.stderr:bluestore allocator fragmentation block 2024-09-21T10:25:44.825 INFO:teuthology.orchestra.run.smithi032.stderr:admin_socket: invalid command 2024-09-21T10:25:44.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:44.881 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:44.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:44.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:44.983 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:45.025 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:45.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:45.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:45.127 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:45.178 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:45.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:45.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:45.279 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:45.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:45.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:45.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:45.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 72 16 114686 114670 99.5256 58.375 0.00828398 0.0100429 2024-09-21T10:25:45.422 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:45.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:45.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:45.569 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:45.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:45.614 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:45.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:45.715 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:45.745 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:45.746 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:25:45.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:45.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:45.848 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:45.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:45.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:45.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:45.921 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:45 smithi045 ceph-mon[95976]: pgmap v1183: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 27672/332055 objects degraded (8.334%) 2024-09-21T10:25:45.972 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:45 smithi032 ceph-mon[105748]: pgmap v1183: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 27672/332055 objects degraded (8.334%) 2024-09-21T10:25:45.972 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:45 smithi032 ceph-mon[101299]: pgmap v1183: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 27672/332055 objects degraded (8.334%) 2024-09-21T10:25:45.998 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:46.009 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:25:46.009 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-0, ['dump_ops_in_flight'] 2024-09-21T10:25:46.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:46.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:46.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:46.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:46.195 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:46.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:46.296 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:46.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 73 16 115616 115600 98.9584 58.125 0.02076 0.0101001 2024-09-21T10:25:46.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:46.335 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:46.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:46.436 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:46.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:46.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:46.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:46.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:46.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:46.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:46.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:46 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[271261]: 2024-09-21T10:25:46.233+0000 7ff59ea3f640 -1 osd.0 382 log_to_monitors true 2024-09-21T10:25:46.722 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:46.754 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:46.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:46.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:46.856 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:46.883 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:25:46.883 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:25:46.892 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:46.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:46.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:46.976 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:46 smithi045 ceph-mon[95976]: from='osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:25:46.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:47.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:47.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:47.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:47.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:47.183 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:47.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:47.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:46 smithi032 ceph-mon[105748]: from='osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:25:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:46 smithi032 ceph-mon[101299]: from='osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:25:47.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:25:46 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[271261]: 2024-09-21T10:25:46.804+0000 7ff5927e1640 -1 osd.0 388 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:25:47.285 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:47.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 74 16 116653 116637 98.4969 64.8125 0.00511051 0.0101475 2024-09-21T10:25:47.329 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:47.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:47.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:47.431 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:47.467 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:47.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:47.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:47.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:47.604 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:47.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:47.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:47.706 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:47.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:25:47.746 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:47.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:47.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:47.848 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:47.871 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:47.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:47.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[105748]: pgmap v1184: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 28564/343068 objects degraded (8.326%) 2024-09-21T10:25:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[105748]: from='osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:25:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[105748]: osdmap e388: 8 total, 7 up, 8 in 2024-09-21T10:25:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[105748]: from='osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:25:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 28564/343068 objects degraded (8.326%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[101299]: pgmap v1184: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 28564/343068 objects degraded (8.326%) 2024-09-21T10:25:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[101299]: from='osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:25:47.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[101299]: osdmap e388: 8 total, 7 up, 8 in 2024-09-21T10:25:47.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[101299]: from='osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:25:47.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 28564/343068 objects degraded (8.326%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:47.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:47.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:47 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:47.972 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:47.995 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:47.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:47.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:47 smithi045 ceph-mon[95976]: pgmap v1184: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 28564/343068 objects degraded (8.326%) 2024-09-21T10:25:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:47 smithi045 ceph-mon[95976]: from='osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:25:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:47 smithi045 ceph-mon[95976]: osdmap e388: 8 total, 7 up, 8 in 2024-09-21T10:25:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:47 smithi045 ceph-mon[95976]: from='osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:25:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:47 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 28564/343068 objects degraded (8.326%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:48.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:47 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:48.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:47 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:48.097 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:48.119 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:48.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:48.221 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:48.243 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:48.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:48.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:48.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 75 16 117127 117111 97.5786 29.625 0.0040833 0.0101696 2024-09-21T10:25:48.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:48.371 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:48.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:48.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:48.472 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:48.495 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:48.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:48.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:48.596 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:48.618 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:48.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:48.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:48.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:48.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:48.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:48.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:48.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:48.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:48.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:48.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:48.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:48.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:48.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:48 smithi045 ceph-mon[95976]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:25:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:48 smithi045 ceph-mon[95976]: osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402] boot 2024-09-21T10:25:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:48 smithi045 ceph-mon[95976]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T10:25:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:25:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:49.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:48 smithi045 ceph-mon[95976]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T10:25:49.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:49.124 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:49.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:49.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[105748]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:25:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[105748]: osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402] boot 2024-09-21T10:25:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[105748]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T10:25:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:25:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[105748]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T10:25:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[101299]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:25:49.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[101299]: osd.0 [v2:172.21.15.32:6802/409271402,v1:172.21.15.32:6803/409271402] boot 2024-09-21T10:25:49.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[101299]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T10:25:49.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:25:49.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:25:49.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:49.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:48 smithi032 ceph-mon[101299]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T10:25:49.226 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:49.250 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:49.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:49.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:49.277 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:49.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 76 16 117127 117111 96.2948 0 - 0.0101696 2024-09-21T10:25:49.352 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:49.378 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:49.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:49.479 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:49.505 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:49.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:49.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:49.607 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:49.634 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:49.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:49.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:49.765 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:49.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:49.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:49.823 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:25:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:25:49] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.51.0" 2024-09-21T10:25:49.823 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:49 smithi032 ceph-mon[101299]: pgmap v1187: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 28564/343068 objects degraded (8.326%) 2024-09-21T10:25:49.823 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:49 smithi032 ceph-mon[101299]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T10:25:49.823 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:49 smithi032 ceph-mon[105748]: pgmap v1187: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 28564/343068 objects degraded (8.326%) 2024-09-21T10:25:49.823 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:49 smithi032 ceph-mon[105748]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T10:25:49.867 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:49.890 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:49.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:49.992 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:50.010 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:49 smithi045 ceph-mon[95976]: pgmap v1187: 99 pgs: 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 28564/343068 objects degraded (8.326%) 2024-09-21T10:25:50.010 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:49 smithi045 ceph-mon[95976]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T10:25:50.016 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:50.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:50.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:50.118 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:50.141 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:50.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:50.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:50.243 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:50.266 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:50.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:50.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:50.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 77 16 117127 117111 95.0443 0 - 0.0101696 2024-09-21T10:25:50.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:50.405 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:50.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:50.422 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:25:50.422 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-21T10:25:50.442 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:25:50.442 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:25:50.461 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:25:50.461 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:25:50.480 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:25:50.480 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:25:50.502 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:25:50.502 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:25:50.506 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:50.527 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:25:50.527 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:25:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:50.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:50.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:50.556 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:25:50.556 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:25:50.587 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:25:50.587 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:25:50.650 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:50.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:50.689 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:50.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:50.790 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:50.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:50.833 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:50.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:50.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:50.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:50 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3331219543' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:50.960 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:50 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3331219543' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:50.978 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:50.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:51.010 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:25:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:50 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3331219543' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:51.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:51.127 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:51.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:51.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:51.269 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:51.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:51.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 78 16 117766 117750 94.3378 13.3125 0.0172519 0.0105945 2024-09-21T10:25:51.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:51.409 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:51.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:51.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:51.511 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:51.546 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:51.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:51.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:51.648 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:51.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:51.701 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:51.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:51.802 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:51.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:51.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:51.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:51.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:51.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:51 smithi032 ceph-mon[105748]: pgmap v1190: 99 pgs: 3 activating+undersized+degraded+remapped, 1 remapped+peering, 10 active+undersized, 3 active+undersized+degraded, 82 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s; 13850/350682 objects degraded (3.949%) 2024-09-21T10:25:51.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:51 smithi032 ceph-mon[101299]: pgmap v1190: 99 pgs: 3 activating+undersized+degraded+remapped, 1 remapped+peering, 10 active+undersized, 3 active+undersized+degraded, 82 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s; 13850/350682 objects degraded (3.949%) 2024-09-21T10:25:52.005 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:52.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:52.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:51 smithi045 ceph-mon[95976]: pgmap v1190: 99 pgs: 3 activating+undersized+degraded+remapped, 1 remapped+peering, 10 active+undersized, 3 active+undersized+degraded, 82 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s; 13850/350682 objects degraded (3.949%) 2024-09-21T10:25:52.107 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:52.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:52.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:52.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:52.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:52.287 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:52.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:52.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:52.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 79 16 118604 118588 93.8066 52.375 0.0560914 0.0106546 2024-09-21T10:25:52.389 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:52.422 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:52.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:52.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:52.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:52.559 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:52.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:52.638 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:52.660 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:52.690 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:52.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:52.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:52.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:52.828 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:52.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:52.870 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:52.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:52.889 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:52 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 13850/350682 objects degraded (3.949%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:52.889 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:52 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 13850/350682 objects degraded (3.949%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:52.971 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:53.002 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:53.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:53.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:53.104 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:53.137 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:53.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:53.239 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:52 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 13850/350682 objects degraded (3.949%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:53.286 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:53.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:53.326 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:25:53.325685+0000 min lat: 0.0011678 max lat: 2.79156 avg lat: 0.0107146 2024-09-21T10:25:53.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:25:53.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 80 16 119435 119419 93.2832 51.9375 0.00617431 0.0107146 2024-09-21T10:25:53.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:53.429 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:53.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:53.530 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:53.573 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:53.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:53.585 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:25:53.675 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:53.716 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:53.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:53.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:53.818 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:53.854 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:53.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:53.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:53.956 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:53.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:53.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:54.064 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:53 smithi032 ceph-mon[105748]: pgmap v1191: 99 pgs: 3 activating+undersized+degraded+remapped, 1 remapped+peering, 95 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 30 MiB/s wr, 478 op/s; 13846/351459 objects degraded (3.940%) 2024-09-21T10:25:54.065 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:53 smithi032 ceph-mon[101299]: pgmap v1191: 99 pgs: 3 activating+undersized+degraded+remapped, 1 remapped+peering, 95 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 30 MiB/s wr, 478 op/s; 13846/351459 objects degraded (3.940%) 2024-09-21T10:25:54.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:54.136 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:54.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:54.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:54.238 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:54.270 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:54.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:54.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:53 smithi045 ceph-mon[95976]: pgmap v1191: 99 pgs: 3 activating+undersized+degraded+remapped, 1 remapped+peering, 95 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 30 MiB/s wr, 478 op/s; 13846/351459 objects degraded (3.940%) 2024-09-21T10:25:54.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 81 16 120259 120243 92.7673 51.5 0.0245901 0.0107743 2024-09-21T10:25:54.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:54.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:54.412 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:54.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:54.513 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:54.558 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:54.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:54.660 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:54.697 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:54.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:54.799 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:54.812 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:25:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:54.853 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:54.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:54.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:54.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:54.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:54.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:55.088 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:55.124 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:55.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:54 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4069450332' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:54 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4069450332' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:55.226 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:55.260 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:55.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:54 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4069450332' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:55.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 82 16 121115 121099 92.2884 53.5 0.0587265 0.0108299 2024-09-21T10:25:55.362 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:55.402 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:55.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:55.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:55.504 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:55.539 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:55.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:55.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:55.641 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:55.680 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:55.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:55.782 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:55.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:55.838 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:55.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:55.940 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:55 smithi032 ceph-mon[105748]: pgmap v1192: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 activating+undersized+degraded+remapped, 95 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 32 MiB/s wr, 518 op/s; 18381/353718 objects degraded (5.197%) 2024-09-21T10:25:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:55 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:25:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4069450332' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:55 smithi032 ceph-mon[105748]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T10:25:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:55 smithi032 ceph-mon[101299]: pgmap v1192: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 activating+undersized+degraded+remapped, 95 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 32 MiB/s wr, 518 op/s; 18381/353718 objects degraded (5.197%) 2024-09-21T10:25:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:55 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:25:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4069450332' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:55 smithi032 ceph-mon[101299]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T10:25:55.973 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:55.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:55.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:56.074 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:56.106 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:56.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:56.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:56.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:56.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:56.249 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:56.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:55 smithi045 ceph-mon[95976]: pgmap v1192: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 activating+undersized+degraded+remapped, 95 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 32 MiB/s wr, 518 op/s; 18381/353718 objects degraded (5.197%) 2024-09-21T10:25:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:55 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:25:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4069450332' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:55 smithi045 ceph-mon[95976]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T10:25:56.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 83 16 121970 121954 91.8203 53.4375 0.0223051 0.0108859 2024-09-21T10:25:56.349 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:56.388 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:56.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:56.489 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:56.526 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:56.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:56.628 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:56.659 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:56.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:56.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:56.761 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:56.799 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:56.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:56.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:56.901 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:56.958 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:56.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:57.059 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:57.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:57.097 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:57.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:57.169 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:25:57.169 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:25:57.169 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.169 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:122667 9.9 e392/390 9:90b6ef47:::benchmark_data_smithi032_265250_object122666:head v 392'7743, mlcod=382'2729)", 2024-09-21T10:25:57.169 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:25:57.137245+0000", 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.024068467, 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.024114274000000002, 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137245+0000", 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.170 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137245+0000", 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137247+0000", 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3039999999999999e-06 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:25:57.171 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137292+0000", 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5194000000000001e-05 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137306+0000", 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4494e-05 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137314+0000", 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9759999999999995e-06 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.172 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137626+0000", 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000311446 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137680+0000", 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3902e-05 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:122668 9.9 e392/390 9:9ba66e59:::benchmark_data_smithi032_265250_object122667:head v 392'7744, mlcod=382'2729)", 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:25:57.137346+0000", 2024-09-21T10:25:57.173 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.023967260000000001, 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.024070126000000001, 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137346+0000", 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:25:57.174 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137346+0000", 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137347+0000", 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.7899999999999999e-07 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137385+0000", 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8432000000000001e-05 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.175 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137392+0000", 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6980000000000001e-06 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.137396+0000", 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5029999999999999e-06 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.145790+0000", 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0083937070000000003 2024-09-21T10:25:57.176 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.145813+0000", 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3252999999999999e-05 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:122670 9.9 e392/390 9:9166a9e4:::benchmark_data_smithi032_265250_object122669:head v 392'7745, mlcod=382'2729)", 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:25:57.138187+0000", 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.023126536, 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.023277352000000001, 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:25:57.177 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.138187+0000", 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.138187+0000", 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.178 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.138188+0000", 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2399999999999996e-07 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.138245+0000", 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6912e-05 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.138256+0000", 2024-09-21T10:25:57.179 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1228e-05 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.138263+0000", 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8870000000000003e-06 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.153006+0000", 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.014743361 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:25:57.180 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.153029+0000", 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2048e-05 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:122682 9.5 e392/390 9:aa11206f:::benchmark_data_smithi032_265250_object122681:head v 392'7682, mlcod=384'2926)", 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:25:57.159266+0000", 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.002047947, 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0022458589999999998, 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:25:57.181 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159266+0000", 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159266+0000", 2024-09-21T10:25:57.182 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159267+0000", 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0380000000000001e-06 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159299+0000", 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2555000000000003e-05 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159318+0000", 2024-09-21T10:25:57.183 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8695999999999999e-05 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159326+0000", 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9920000000000007e-06 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159397+0000", 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0549000000000005e-05 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.184 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159422+0000", 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5658000000000001e-05 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:122683 9.8 e392/390 9:1dd20a9f:::benchmark_data_smithi032_265250_object122682:head v 392'7614, mlcod=382'2731)", 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:25:57.159368+0000", 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0019454369999999999, 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0021920220000000001, 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:25:57.185 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159368+0000", 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159368+0000", 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:25:57.186 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159369+0000", 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8599999999999998e-07 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159389+0000", 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0350999999999999e-05 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159397+0000", 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7789999999999996e-06 2024-09-21T10:25:57.187 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159402+0000", 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1449999999999999e-06 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159652+0000", 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024970599999999998 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:25:57.159700+0000", 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8477999999999999e-05 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.188 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:25:57.189 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.189 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:25:57.189 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:25:57.189 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 5 2024-09-21T10:25:57.189 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:25:57.197 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:57.232 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:57.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:57.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:57.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 84 16 122828 122812 91.3655 53.625 0.0306624 0.0109401 2024-09-21T10:25:57.333 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:57.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:57.373 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:57.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:57.474 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:57.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:57.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:57.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:57.613 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:57.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:57.648 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:57.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:57.725 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:25:57.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:57.819 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:57.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:57.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:57.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:57.990 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:57.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:57.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:58.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:58.128 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:57 smithi032 ceph-mon[101299]: pgmap v1194: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 18252/362565 objects degraded (5.034%); 1.0 MiB/s, 16 objects/s recovering 2024-09-21T10:25:58.129 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:57 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 18252/362565 objects degraded (5.034%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:58.129 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:57 smithi032 ceph-mon[105748]: pgmap v1194: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 18252/362565 objects degraded (5.034%); 1.0 MiB/s, 16 objects/s recovering 2024-09-21T10:25:58.129 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:57 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 18252/362565 objects degraded (5.034%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:58.133 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:58.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:58.235 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:58.276 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:58.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:58.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:57 smithi045 ceph-mon[95976]: pgmap v1194: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 18252/362565 objects degraded (5.034%); 1.0 MiB/s, 16 objects/s recovering 2024-09-21T10:25:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:57 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 18252/362565 objects degraded (5.034%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:25:58.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 85 16 123673 123657 90.9119 52.8125 0.0107894 0.0109942 2024-09-21T10:25:58.379 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:58.415 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:58.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:58.517 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:58.554 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:58.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:58.656 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:58.690 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:58.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:58.791 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:58.828 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:25:58.828 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:25:58.828 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:25:58.828 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:25:58.829 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:25:58.840 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:58.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:58.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:58.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:58.973 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:25:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:59.014 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:59.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:25:59.114 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:59.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:59.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:59.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:59.257 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:59.300 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:59.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:59.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:59.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 86 16 124512 124496 90.4645 52.4375 0.0165675 0.0110493 2024-09-21T10:25:59.402 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:59.435 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:25:59.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:59.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:59.537 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:59.574 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:59.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:59.676 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:59.713 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:59.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:59.813 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:59.816 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:25:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:25:59] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.51.0" 2024-09-21T10:25:59.851 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:59.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:59.952 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:59.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:25:59.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:59.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:00.091 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:00.127 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:00.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:00.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:00.149 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:25:59 smithi032 ceph-mon[101299]: pgmap v1195: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s; 18252/362565 objects degraded (5.034%); 926 KiB/s, 14 objects/s recovering 2024-09-21T10:26:00.149 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:25:59 smithi032 ceph-mon[105748]: pgmap v1195: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s; 18252/362565 objects degraded (5.034%); 926 KiB/s, 14 objects/s recovering 2024-09-21T10:26:00.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:00.265 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:00.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:25:59 smithi045 ceph-mon[95976]: pgmap v1195: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s; 18252/362565 objects degraded (5.034%); 926 KiB/s, 14 objects/s recovering 2024-09-21T10:26:00.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 87 16 125345 125329 90.0231 52.0625 0.0135417 0.0111023 2024-09-21T10:26:00.367 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:00.413 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:00.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:00.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:00.515 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:00.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:00.552 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:00.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:00.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:00.689 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:00.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:00.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:00.790 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:00.817 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:00.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:00.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:00.918 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:00.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:00.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:00.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:01.057 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:01.094 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:01.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:01.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:01.196 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:01.237 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:01.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:01.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:01.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 88 16 126210 126194 89.6143 54.0625 0.0128716 0.0111533 2024-09-21T10:26:01.338 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:01.378 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:01.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:01.480 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:01.513 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:01.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:01.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:01.615 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:01.649 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:01.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:01.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:01.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:26:01.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:01.765 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:01.807 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:01.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:01.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:01.910 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:01.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:01 smithi032 ceph-mon[101299]: pgmap v1196: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 18113/373800 objects degraded (4.846%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:26:01.964 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:01 smithi032 ceph-mon[105748]: pgmap v1196: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 18113/373800 objects degraded (4.846%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:26:01.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:01.965 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:01.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:02.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:02.120 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:02.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:02.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:02.222 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:02.261 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:02.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:01 smithi045 ceph-mon[95976]: pgmap v1196: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 18113/373800 objects degraded (4.846%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:26:02.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 89 16 127034 127018 89.186 51.5 0.011535 0.0112071 2024-09-21T10:26:02.364 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:02.406 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:02.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:02.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:02.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:02.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:02.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:02.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:02.642 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:02.680 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:02.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:02.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:02.782 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:02.815 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:02.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:02.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:02.917 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:02.945 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:02.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:03.038 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:02 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 18113/373800 objects degraded (4.846%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:03.038 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:02 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 18113/373800 objects degraded (4.846%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:03.047 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:03.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:03.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:03.201 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:03.233 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:03.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:02 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 18113/373800 objects degraded (4.846%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:03.327 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 90 1 127799 127798 88.7367 48.75 0.0118059 0.0112644 2024-09-21T10:26:03.335 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:03.359 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:03.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:03.461 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:03.483 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:03.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:03.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:03.585 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:03.585 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 1, 0, 4] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 6, 7, 5, 0] 2024-09-21T10:26:03.586 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:26:03.586 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:26:03.586 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:26:03.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:03.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:03.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:03.733 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:03.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:03.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:03.835 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:03.857 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:03.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:03.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:03.982 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:03.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:03.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:04.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:04.106 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:04.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:04.132 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:03 smithi032 ceph-mon[101299]: pgmap v1197: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s; 18113/375387 objects degraded (4.825%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:26:04.132 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:04.132 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:03 smithi032 ceph-mon[105748]: pgmap v1197: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s; 18113/375387 objects degraded (4.825%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:26:04.133 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:04.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:04.231 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:04.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:04.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:03 smithi045 ceph-mon[95976]: pgmap v1197: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s; 18113/375387 objects degraded (4.825%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:26:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total time run: 90.0138 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total writes made: 127799 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Write size: 65536 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object size: 65536 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Bandwidth (MB/sec): 88.7357 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Bandwidth: 41.2449 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max bandwidth (MB/sec): 233.688 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average IOPS: 1419 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev IOPS: 659.918 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max IOPS: 3739 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min IOPS: 0 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average Latency(s): 0.0112645 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Latency(s): 0.0336061 2024-09-21T10:26:04.328 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max latency(s): 2.79156 2024-09-21T10:26:04.329 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min latency(s): 0.0011678 2024-09-21T10:26:04.332 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:04.333 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:26:04.366 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:04.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:04.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:04.468 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:04.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:04.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:04.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:04.598 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:04.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:04.637 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:04.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:04.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:04.788 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:04.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:04.889 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:04.919 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:04.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:04.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:05.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:05.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:05.067 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:05.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:05.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:05.190 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:26:05.191 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:26:05.204 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:05.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:05.225 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:26:05.225 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:26:05.261 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:26:05.262 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:26:05.306 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:05.311 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:26:05.311 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:26:05.346 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:26:05.346 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:26:05.365 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:05.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:05.377 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:26:05.378 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:26:05.411 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:26:05.411 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:26:05.444 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:26:05.444 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:26:05.466 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:05.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:05.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:05.600 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:05.629 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:05.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:05.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:05.730 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:05.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:05.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:05.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:05.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:05.900 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:05.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:06.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:06.038 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:06.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:06.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:06.095 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:06.140 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:06.186 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:06.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:06.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:06.197 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:05 smithi032 ceph-mon[105748]: pgmap v1198: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 18113/377898 objects degraded (4.793%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:26:06.197 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4223389260' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:06.197 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:05 smithi032 ceph-mon[101299]: pgmap v1198: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 18113/377898 objects degraded (4.793%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:26:06.197 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4223389260' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:05 smithi045 ceph-mon[95976]: pgmap v1198: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 18113/377898 objects degraded (4.793%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:26:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4223389260' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:06.287 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:06.322 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:06.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:06.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:06.423 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:06.459 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:06.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:06.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:06.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:06.602 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:06.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:06.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:06.704 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:06.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:06.742 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:06.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:06.843 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:06.892 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:06.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:06.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:07.021 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:07.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:07.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:07.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:07.134 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:06 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 18113/377898 objects degraded (4.793%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:07.135 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:06 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 18113/377898 objects degraded (4.793%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:07.160 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:07.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:07.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:07.276 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:06 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 18113/377898 objects degraded (4.793%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:07.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:07.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:07.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:07.414 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:07.445 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:07.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:07.547 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:07.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:07.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:07.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:07.737 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:07.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:07.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:07.840 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:07.875 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:07.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:07.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:07.884 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:26:07.977 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:08.013 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:08.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:08.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:08.142 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:07 smithi032 ceph-mon[101299]: pgmap v1199: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 86 B/s rd, 53 MiB/s wr, 898 op/s; 17903/381516 objects degraded (4.693%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:26:08.142 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:07 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3169385267' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:26:08.142 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:26:08.142 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:07 smithi032 ceph-mon[105748]: pgmap v1199: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 86 B/s rd, 53 MiB/s wr, 898 op/s; 17903/381516 objects degraded (4.693%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:26:08.142 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:07 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3169385267' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:26:08.142 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:26:08.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:08.201 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:08.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:08.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:07 smithi045 ceph-mon[95976]: pgmap v1199: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 86 B/s rd, 53 MiB/s wr, 898 op/s; 17903/381516 objects degraded (4.693%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:26:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:07 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3169385267' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:26:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:26:08.303 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:08.342 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:08.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:08.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:08.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:08.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:08.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:08.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:08.578 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:26:08.580 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:08.678 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:08.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:08.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:08.780 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:08.812 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:08.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:08.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:08.913 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:08.947 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:08.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:09.049 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:09.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:09.084 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:09.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:09.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:09.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:08 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:09.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:08 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:26:09.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:08 smithi032 ceph-mon[101299]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T10:26:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:08 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:08 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:26:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:08 smithi032 ceph-mon[105748]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T10:26:09.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:09.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:09.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:09.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:08 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:09.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:08 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:26:09.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:08 smithi045 ceph-mon[95976]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T10:26:09.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:09.374 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:09.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:09.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:09.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:09.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:09.522 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:09.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:09.623 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:09.685 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:09.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:09.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:09.787 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:09.810 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:09.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:09.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:09.911 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:09.956 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:26:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:26:09] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.51.0" 2024-09-21T10:26:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:09.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:10.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:10.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:10.093 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:10.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:10.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:10.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:10.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:10.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:10.364 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:10.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:10.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[105748]: pgmap v1201: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 762 op/s; 17903/381516 objects degraded (4.693%); 2.2 MiB/s, 34 objects/s recovering 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[101299]: pgmap v1201: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 762 op/s; 17903/381516 objects degraded (4.693%); 2.2 MiB/s, 34 objects/s recovering 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:26:10.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:26:10.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:26:10.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:26:10.466 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:10.496 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:10.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:10.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:10 smithi045 ceph-mon[95976]: pgmap v1201: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 762 op/s; 17903/381516 objects degraded (4.693%); 2.2 MiB/s, 34 objects/s recovering 2024-09-21T10:26:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:26:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:26:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:26:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:26:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:26:10.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:10.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:10.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:10.733 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:10.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:10.779 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:10.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:10.880 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:10.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:10.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:10.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:11.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:11.065 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:11.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:11.167 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:11.203 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:11.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:11.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:11.305 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:11.332 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:26:11.332 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:26:11.367 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:11.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:11.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:11.469 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:11.504 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:11.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:11.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:11.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:11.641 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:11.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:11.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:11.743 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:11.783 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:11.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:11.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:11.884 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:11.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:11.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:11.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:12.030 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:12.068 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:12.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:12.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:12.170 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:12.204 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:26:12.211 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:12.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:12.313 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:12.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:12.349 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:12.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:12.450 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:12 smithi032 ceph-mon[101299]: pgmap v1202: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 755 op/s; 17752/370488 objects degraded (4.792%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:12.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:12 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 17752/370488 objects degraded (4.792%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:12.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:12 smithi032 ceph-mon[105748]: pgmap v1202: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 755 op/s; 17752/370488 objects degraded (4.792%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:12.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:12 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 17752/370488 objects degraded (4.792%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:12.487 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:12.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:12.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:12 smithi045 ceph-mon[95976]: pgmap v1202: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 755 op/s; 17752/370488 objects degraded (4.792%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:12 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 17752/370488 objects degraded (4.792%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:12.561 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:12.589 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:12.661 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:12.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:12.763 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:12.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:12.807 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:12.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:12.907 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:12.958 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:12.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:12.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:13.031 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:26:13.059 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:13.103 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:13.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:13.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:13.187 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:13.205 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:13.237 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:13.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:13.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:13.339 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:13.391 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:13.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:13.492 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:13.528 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:13.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:13.630 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:13.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:13.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:13.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:13.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:13.818 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:13.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:13.920 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:13.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:13.960 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:13.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:14.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:14.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:14.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:14.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:14.245 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:14.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:14.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:14.346 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:14.382 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:14.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:14 smithi032 ceph-mon[101299]: pgmap v1203: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 755 op/s; 17752/368907 objects degraded (4.812%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:14 smithi032 ceph-mon[105748]: pgmap v1203: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 755 op/s; 17752/368907 objects degraded (4.812%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:14.484 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:14.507 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:14 smithi045 ceph-mon[95976]: pgmap v1203: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 755 op/s; 17752/368907 objects degraded (4.812%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:14.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:14.527 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:14.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:14.627 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:14.667 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:14.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:14.679 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:26:14.679 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:26:14.769 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:14.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:14.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:14.912 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:14.943 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:14.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:15.045 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:15.097 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:15.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:15.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:15.114 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:26:15.116 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:26:15.199 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:15.254 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:15.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:15.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:15.356 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:15.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:15.401 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:15.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:15.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:15.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:15.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:15.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:15.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:15.693 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:15.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:15.795 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:15.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:15.833 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:15.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:15.934 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:15.967 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:15.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:15.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:16.069 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:16.103 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:16.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:16.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:16.204 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:16.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:16.242 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:16.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:16.283 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1204,"stamp":"2024-09-21T10:26:14.594107+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7990351869,"num_objects":121879,"num_object_clones":0,"num_object_copies":365637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17752,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":121879,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":134657,"num_write_kb":8191762,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":915,"num_bytes_recovered":58070661,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123623,"ondisk_log_size":123623,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":27331776,"kb_used_data":22408440,"kb_used_omap":45,"kb_used_meta":4923026,"kb_avail":722629440,"statfs":{"total":767960285184,"available":739972546560,"internally_reserved":0,"allocated":22946242560,"data_stored":22930077535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46417,"internal_metadata":5041179311},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-267911144,"num_objects":-4087,"num_object_clones":0,"num_object_copies":-12261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-361,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4087,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7811,"num_write_kb":119105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003085"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":664,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.885114+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:26:07.885114+0000","last_peered":"2024-09-21T10:26:07.885114+0000","last_clean":"2024-09-21T10:26:07.885114+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:26:07.885114+0000","last_undegraded":"2024-09-21T10:26:07.885114+0000","last_fullsized":"2024-09-21T10:26:07.885114+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":397,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.808210+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:25:54.808210+0000","last_peered":"2024-09-21T10:25:54.808210+0000","last_clean":"2024-09-21T10:25:54.808210+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:25:54.808210+0000","last_undegraded":"2024-09-21T10:25:54.808210+0000","last_fullsized":"2024-09-21T10:25:54.808210+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":93,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.805488+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:25:54.805488+0000","last_peered":"2024-09-21T10:25:54.805488+0000","last_clean":"2024-09-21T10:25:54.805488+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:25:54.805488+0000","last_undegraded":"2024-09-21T10:25:54.805488+0000","last_fullsized":"2024-09-21T10:25:54.805488+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":31,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.891828+0000","last_change":"2024-09-21T10:25:48.765404+0000","last_active":"2024-09-21T10:26:07.891828+0000","last_peered":"2024-09-21T10:26:07.891828+0000","last_clean":"2024-09-21T10:26:07.891828+0000","last_became_active":"2024-09-21T10:25:48.764230+0000","last_became_peered":"2024-09-21T10:25:48.764230+0000","last_unstale":"2024-09-21T10:26:07.891828+0000","last_undegraded":"2024-09-21T10:26:07.891828+0000","last_fullsized":"2024-09-21T10:26:07.891828+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:38.551941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":32,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892271+0000","last_change":"2024-09-21T10:25:48.764391+0000","last_active":"2024-09-21T10:26:07.892271+0000","last_peered":"2024-09-21T10:26:07.892271+0000","last_clean":"2024-09-21T10:26:07.892271+0000","last_became_active":"2024-09-21T10:25:48.763375+0000","last_became_peered":"2024-09-21T10:25:48.763375+0000","last_unstale":"2024-09-21T10:26:07.892271+0000","last_undegraded":"2024-09-21T10:26:07.892271+0000","last_fullsized":"2024-09-21T10:26:07.892271+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:03.457716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":368,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.807579+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:25:54.807579+0000","last_peered":"2024-09-21T10:25:54.807579+0000","last_clean":"2024-09-21T10:25:54.807579+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:25:54.807579+0000","last_undegraded":"2024-09-21T10:25:54.807579+0000","last_fullsized":"2024-09-21T10:25:54.807579+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":685,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.889664+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:26:07.889664+0000","last_peered":"2024-09-21T10:26:07.889664+0000","last_clean":"2024-09-21T10:26:07.889664+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:26:07.889664+0000","last_undegraded":"2024-09-21T10:26:07.889664+0000","last_fullsized":"2024-09-21T10:26:07.889664+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":464,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.812072+0000","last_change":"2024-09-21T10:25:48.752712+0000","last_active":"2024-09-21T10:25:54.812072+0000","last_peered":"2024-09-21T10:25:54.812072+0000","last_clean":"2024-09-21T10:25:54.812072+0000","last_became_active":"2024-09-21T10:25:48.752204+0000","last_became_peered":"2024-09-21T10:25:48.752204+0000","last_unstale":"2024-09-21T10:25:54.812072+0000","last_undegraded":"2024-09-21T10:25:54.812072+0000","last_fullsized":"2024-09-21T10:25:54.812072+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:40.068520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"393'8329","reported_seq":9624,"reported_epoch":393,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:26:10.350850+0000","last_change":"2024-09-21T10:25:50.495922+0000","last_active":"2024-09-21T10:26:10.350850+0000","last_peered":"2024-09-21T10:26:10.350850+0000","last_clean":"2024-09-21T10:24:52.083026+0000","last_became_active":"2024-09-21T10:25:50.476041+0000","last_became_peered":"2024-09-21T10:25:50.476041+0000","last_unstale":"2024-09-21T10:26:10.350850+0000","last_undegraded":"2024-09-21T10:25:49.844484+0000","last_fullsized":"2024-09-21T10:25:49.744829+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8329,"log_dups_size":0,"ondisk_log_size":8329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:56.903795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":503775232,"num_objects":7687,"num_object_clones":0,"num_object_copies":23061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3948,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8329,"num_write_kb":512512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":628,"num_bytes_recovered":41156608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":3739}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":99,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:26:05.098268+0000","last_change":"2024-09-21T10:25:48.755730+0000","last_active":"2024-09-21T10:26:05.098268+0000","last_peered":"2024-09-21T10:26:05.098268+0000","last_clean":"2024-09-21T10:26:05.098268+0000","last_became_active":"2024-09-21T10:25:48.755344+0000","last_became_peered":"2024-09-21T10:25:48.755344+0000","last_unstale":"2024-09-21T10:26:05.098268+0000","last_undegraded":"2024-09-21T10:26:05.098268+0000","last_fullsized":"2024-09-21T10:26:05.098268+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:37.725090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":101,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.804654+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:25:54.804654+0000","last_peered":"2024-09-21T10:25:54.804654+0000","last_clean":"2024-09-21T10:25:54.804654+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:25:54.804654+0000","last_undegraded":"2024-09-21T10:25:54.804654+0000","last_fullsized":"2024-09-21T10:25:54.804654+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"393'8289","reported_seq":8330,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:10.612908+0000","last_change":"2024-09-21T10:24:25.829780+0000","last_active":"2024-09-21T10:26:10.612908+0000","last_peered":"2024-09-21T10:26:10.612908+0000","last_clean":"2024-09-21T10:26:10.612908+0000","last_became_active":"2024-09-21T10:24:25.829509+0000","last_became_peered":"2024-09-21T10:24:25.829509+0000","last_unstale":"2024-09-21T10:26:10.612908+0000","last_undegraded":"2024-09-21T10:26:10.612908+0000","last_fullsized":"2024-09-21T10:26:10.612908+0000","mapping_epoch":378,"log_start":"382'900","ondisk_log_start":"382'900","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7389,"log_dups_size":0,"ondisk_log_size":7389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:30.108791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":501940224,"num_objects":7659,"num_object_clones":0,"num_object_copies":22977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8289,"num_write_kb":510336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":71,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:26:05.098241+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:26:05.098241+0000","last_peered":"2024-09-21T10:26:05.098241+0000","last_clean":"2024-09-21T10:26:05.098241+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:26:05.098241+0000","last_undegraded":"2024-09-21T10:26:05.098241+0000","last_fullsized":"2024-09-21T10:26:05.098241+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819367+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:25:54.819367+0000","last_peered":"2024-09-21T10:25:54.819367+0000","last_clean":"2024-09-21T10:25:54.819367+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:25:54.819367+0000","last_undegraded":"2024-09-21T10:25:54.819367+0000","last_fullsized":"2024-09-21T10:25:54.819367+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"393'8323","reported_seq":8347,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:10.321096+0000","last_change":"2024-09-21T10:24:25.825494+0000","last_active":"2024-09-21T10:26:10.321096+0000","last_peered":"2024-09-21T10:26:10.321096+0000","last_clean":"2024-09-21T10:26:10.321096+0000","last_became_active":"2024-09-21T10:24:25.825196+0000","last_became_peered":"2024-09-21T10:24:25.825196+0000","last_unstale":"2024-09-21T10:26:10.321096+0000","last_undegraded":"2024-09-21T10:26:10.321096+0000","last_fullsized":"2024-09-21T10:26:10.321096+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8323,"log_dups_size":0,"ondisk_log_size":8323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:50.985537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":504692736,"num_objects":7701,"num_object_clones":0,"num_object_copies":23103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8323,"num_write_kb":512768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":392,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.808050+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:25:54.808050+0000","last_peered":"2024-09-21T10:25:54.808050+0000","last_clean":"2024-09-21T10:25:54.808050+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:25:54.808050+0000","last_undegraded":"2024-09-21T10:25:54.808050+0000","last_fullsized":"2024-09-21T10:25:54.808050+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":412,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.812205+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:25:54.812205+0000","last_peered":"2024-09-21T10:25:54.812205+0000","last_clean":"2024-09-21T10:25:54.812205+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:25:54.812205+0000","last_undegraded":"2024-09-21T10:25:54.812205+0000","last_fullsized":"2024-09-21T10:25:54.812205+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"393'8085","reported_seq":8109,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:10.320382+0000","last_change":"2024-09-21T10:24:25.846770+0000","last_active":"2024-09-21T10:26:10.320382+0000","last_peered":"2024-09-21T10:26:10.320382+0000","last_clean":"2024-09-21T10:26:10.320382+0000","last_became_active":"2024-09-21T10:24:25.846530+0000","last_became_peered":"2024-09-21T10:24:25.846530+0000","last_unstale":"2024-09-21T10:26:10.320382+0000","last_undegraded":"2024-09-21T10:26:10.320382+0000","last_fullsized":"2024-09-21T10:26:10.320382+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8085,"log_dups_size":0,"ondisk_log_size":8085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:50.827496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":491716608,"num_objects":7503,"num_object_clones":0,"num_object_copies":22509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8085,"num_write_kb":498816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":326,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.804846+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:25:54.804846+0000","last_peered":"2024-09-21T10:25:54.804846+0000","last_clean":"2024-09-21T10:25:54.804846+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:25:54.804846+0000","last_undegraded":"2024-09-21T10:25:54.804846+0000","last_fullsized":"2024-09-21T10:25:54.804846+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":222,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.893333+0000","last_change":"2024-09-21T10:25:48.763096+0000","last_active":"2024-09-21T10:26:07.893333+0000","last_peered":"2024-09-21T10:26:07.893333+0000","last_clean":"2024-09-21T10:26:07.893333+0000","last_became_active":"2024-09-21T10:25:48.761327+0000","last_became_peered":"2024-09-21T10:25:48.761327+0000","last_unstale":"2024-09-21T10:26:07.893333+0000","last_undegraded":"2024-09-21T10:26:07.893333+0000","last_fullsized":"2024-09-21T10:26:07.893333+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:23.154333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"393'8599","reported_seq":8638,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:14.161956+0000","last_change":"2024-09-21T10:24:25.846977+0000","last_active":"2024-09-21T10:26:14.161956+0000","last_peered":"2024-09-21T10:26:14.161956+0000","last_clean":"2024-09-21T10:26:14.161956+0000","last_became_active":"2024-09-21T10:24:25.846670+0000","last_became_peered":"2024-09-21T10:24:25.846670+0000","last_unstale":"2024-09-21T10:26:14.161956+0000","last_undegraded":"2024-09-21T10:26:14.161956+0000","last_fullsized":"2024-09-21T10:26:14.161956+0000","mapping_epoch":378,"log_start":"382'1700","ondisk_log_start":"382'1700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6899,"log_dups_size":0,"ondisk_log_size":6899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:28.387249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":491061248,"num_objects":7493,"num_object_clones":0,"num_object_copies":22479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8599,"num_write_kb":514944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":678,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:26:00.090760+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:26:00.090760+0000","last_peered":"2024-09-21T10:26:00.090760+0000","last_clean":"2024-09-21T10:26:00.090760+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:26:00.090760+0000","last_undegraded":"2024-09-21T10:26:00.090760+0000","last_fullsized":"2024-09-21T10:26:00.090760+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":725,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.883561+0000","last_change":"2024-09-21T10:25:48.755999+0000","last_active":"2024-09-21T10:26:07.883561+0000","last_peered":"2024-09-21T10:26:07.883561+0000","last_clean":"2024-09-21T10:26:07.883561+0000","last_became_active":"2024-09-21T10:25:48.754257+0000","last_became_peered":"2024-09-21T10:25:48.754257+0000","last_unstale":"2024-09-21T10:26:07.883561+0000","last_undegraded":"2024-09-21T10:26:07.883561+0000","last_fullsized":"2024-09-21T10:26:07.883561+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:06.025116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819726+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:25:54.819726+0000","last_peered":"2024-09-21T10:25:54.819726+0000","last_clean":"2024-09-21T10:25:54.819726+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:25:54.819726+0000","last_undegraded":"2024-09-21T10:25:54.819726+0000","last_fullsized":"2024-09-21T10:25:54.819726+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"393'8438","reported_seq":8479,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:10.627196+0000","last_change":"2024-09-21T10:24:25.847167+0000","last_active":"2024-09-21T10:26:10.627196+0000","last_peered":"2024-09-21T10:26:10.627196+0000","last_clean":"2024-09-21T10:26:10.627196+0000","last_became_active":"2024-09-21T10:24:25.846878+0000","last_became_peered":"2024-09-21T10:24:25.846878+0000","last_unstale":"2024-09-21T10:26:10.627196+0000","last_undegraded":"2024-09-21T10:26:10.627196+0000","last_fullsized":"2024-09-21T10:26:10.627196+0000","mapping_epoch":378,"log_start":"382'1100","ondisk_log_start":"382'1100","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7338,"log_dups_size":0,"ondisk_log_size":7338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:32.840244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":506724352,"num_objects":7732,"num_object_clones":0,"num_object_copies":23196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8438,"num_write_kb":517440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":450,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.804578+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:25:54.804578+0000","last_peered":"2024-09-21T10:25:54.804578+0000","last_clean":"2024-09-21T10:25:54.804578+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:25:54.804578+0000","last_undegraded":"2024-09-21T10:25:54.804578+0000","last_fullsized":"2024-09-21T10:25:54.804578+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":402,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.812264+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:25:54.812264+0000","last_peered":"2024-09-21T10:25:54.812264+0000","last_clean":"2024-09-21T10:25:54.812264+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:25:54.812264+0000","last_undegraded":"2024-09-21T10:25:54.812264+0000","last_fullsized":"2024-09-21T10:25:54.812264+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819812+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:25:54.819812+0000","last_peered":"2024-09-21T10:25:54.819812+0000","last_clean":"2024-09-21T10:25:54.819812+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:25:54.819812+0000","last_undegraded":"2024-09-21T10:25:54.819812+0000","last_fullsized":"2024-09-21T10:25:54.819812+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"393'8335","reported_seq":8359,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:10.085153+0000","last_change":"2024-09-21T10:24:25.828673+0000","last_active":"2024-09-21T10:26:10.085153+0000","last_peered":"2024-09-21T10:26:10.085153+0000","last_clean":"2024-09-21T10:26:10.085153+0000","last_became_active":"2024-09-21T10:24:25.828433+0000","last_became_peered":"2024-09-21T10:24:25.828433+0000","last_unstale":"2024-09-21T10:26:10.085153+0000","last_undegraded":"2024-09-21T10:26:10.085153+0000","last_fullsized":"2024-09-21T10:26:10.085153+0000","mapping_epoch":378,"log_start":"382'1300","ondisk_log_start":"382'1300","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7035,"log_dups_size":0,"ondisk_log_size":7035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:07.654019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":503644160,"num_objects":7685,"num_object_clones":0,"num_object_copies":23055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8335,"num_write_kb":512640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819282+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:25:54.819282+0000","last_peered":"2024-09-21T10:25:54.819282+0000","last_clean":"2024-09-21T10:25:54.819282+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:25:54.819282+0000","last_undegraded":"2024-09-21T10:25:54.819282+0000","last_fullsized":"2024-09-21T10:25:54.819282+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.820020+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:25:54.820020+0000","last_peered":"2024-09-21T10:25:54.820020+0000","last_clean":"2024-09-21T10:25:54.820020+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:25:54.820020+0000","last_undegraded":"2024-09-21T10:25:54.820020+0000","last_fullsized":"2024-09-21T10:25:54.820020+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"393'8575","reported_seq":8616,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:14.092914+0000","last_change":"2024-09-21T10:24:25.847020+0000","last_active":"2024-09-21T10:26:14.092914+0000","last_peered":"2024-09-21T10:26:14.092914+0000","last_clean":"2024-09-21T10:26:14.092914+0000","last_became_active":"2024-09-21T10:24:25.846622+0000","last_became_peered":"2024-09-21T10:24:25.846622+0000","last_unstale":"2024-09-21T10:26:14.092914+0000","last_undegraded":"2024-09-21T10:26:14.092914+0000","last_fullsized":"2024-09-21T10:26:14.092914+0000","mapping_epoch":378,"log_start":"382'1700","ondisk_log_start":"382'1700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6875,"log_dups_size":0,"ondisk_log_size":6875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:54:04.668262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":491585536,"num_objects":7501,"num_object_clones":0,"num_object_copies":22503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8575,"num_write_kb":514432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":628,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.804776+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:25:54.804776+0000","last_peered":"2024-09-21T10:25:54.804776+0000","last_clean":"2024-09-21T10:25:54.804776+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:25:54.804776+0000","last_undegraded":"2024-09-21T10:25:54.804776+0000","last_fullsized":"2024-09-21T10:25:54.804776+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":693,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.886931+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:26:07.886931+0000","last_peered":"2024-09-21T10:26:07.886931+0000","last_clean":"2024-09-21T10:26:07.886931+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:26:07.886931+0000","last_undegraded":"2024-09-21T10:26:07.886931+0000","last_fullsized":"2024-09-21T10:26:07.886931+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":385,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.893081+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:26:07.893081+0000","last_peered":"2024-09-21T10:26:07.893081+0000","last_clean":"2024-09-21T10:26:07.893081+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:26:07.893081+0000","last_undegraded":"2024-09-21T10:26:07.893081+0000","last_fullsized":"2024-09-21T10:26:07.893081+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"393'8458","reported_seq":8511,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:26:14.142566+0000","last_change":"2024-09-21T10:25:50.496957+0000","last_active":"2024-09-21T10:26:14.142566+0000","last_peered":"2024-09-21T10:26:14.142566+0000","last_clean":"2024-09-21T10:24:52.081220+0000","last_became_active":"2024-09-21T10:25:50.482809+0000","last_became_peered":"2024-09-21T10:25:50.482809+0000","last_unstale":"2024-09-21T10:26:14.142566+0000","last_undegraded":"2024-09-21T10:25:49.768122+0000","last_fullsized":"2024-09-21T10:25:49.744590+0000","mapping_epoch":390,"log_start":"382'1600","ondisk_log_start":"382'1600","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6858,"log_dups_size":0,"ondisk_log_size":6858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486932480,"num_objects":7430,"num_object_clones":0,"num_object_copies":22290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4535,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8458,"num_write_kb":508416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":2895}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":677,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.807253+0000","last_change":"2024-09-21T10:25:48.754725+0000","last_active":"2024-09-21T10:25:54.807253+0000","last_peered":"2024-09-21T10:25:54.807253+0000","last_clean":"2024-09-21T10:25:54.807253+0000","last_became_active":"2024-09-21T10:25:48.754341+0000","last_became_peered":"2024-09-21T10:25:48.754341+0000","last_unstale":"2024-09-21T10:25:54.807253+0000","last_undegraded":"2024-09-21T10:25:54.807253+0000","last_fullsized":"2024-09-21T10:25:54.807253+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:25.191555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:25.191555+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:08.981054+0000","scrub_duration":0.0015044590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":705,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.806070+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:25:54.806070+0000","last_peered":"2024-09-21T10:25:54.806070+0000","last_clean":"2024-09-21T10:25:54.806070+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:25:54.806070+0000","last_undegraded":"2024-09-21T10:25:54.806070+0000","last_fullsized":"2024-09-21T10:25:54.806070+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":624,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.882739+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:26:07.882739+0000","last_peered":"2024-09-21T10:26:07.882739+0000","last_clean":"2024-09-21T10:26:07.882739+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:26:07.882739+0000","last_undegraded":"2024-09-21T10:26:07.882739+0000","last_fullsized":"2024-09-21T10:26:07.882739+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":369,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:55.063638+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:25:55.063638+0000","last_peered":"2024-09-21T10:25:55.063638+0000","last_clean":"2024-09-21T10:25:55.063638+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:25:55.063638+0000","last_undegraded":"2024-09-21T10:25:55.063638+0000","last_fullsized":"2024-09-21T10:25:55.063638+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"393'8174","reported_seq":8214,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:26:10.337763+0000","last_change":"2024-09-21T10:25:50.516299+0000","last_active":"2024-09-21T10:26:10.337763+0000","last_peered":"2024-09-21T10:26:10.337763+0000","last_clean":"2024-09-21T10:24:52.080442+0000","last_became_active":"2024-09-21T10:25:50.497794+0000","last_became_peered":"2024-09-21T10:25:50.497794+0000","last_unstale":"2024-09-21T10:26:10.337763+0000","last_undegraded":"2024-09-21T10:25:49.773994+0000","last_fullsized":"2024-09-21T10:25:49.744196+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8174,"log_dups_size":0,"ondisk_log_size":8174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:02.445442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":495190016,"num_objects":7556,"num_object_clones":0,"num_object_copies":22668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4583,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8174,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":2973}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":640,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.893216+0000","last_change":"2024-09-21T10:24:27.359111+0000","last_active":"2024-09-21T10:26:07.893216+0000","last_peered":"2024-09-21T10:26:07.893216+0000","last_clean":"2024-09-21T10:26:07.893216+0000","last_became_active":"2024-09-21T10:24:24.825850+0000","last_became_peered":"2024-09-21T10:24:24.825850+0000","last_unstale":"2024-09-21T10:26:07.893216+0000","last_undegraded":"2024-09-21T10:26:07.893216+0000","last_fullsized":"2024-09-21T10:26:07.893216+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:27.359065+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:24:27.359065+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:03.723058+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1049,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.807347+0000","last_change":"2024-09-21T10:25:48.755667+0000","last_active":"2024-09-21T10:25:54.807347+0000","last_peered":"2024-09-21T10:25:54.807347+0000","last_clean":"2024-09-21T10:25:54.807347+0000","last_became_active":"2024-09-21T10:25:48.755060+0000","last_became_peered":"2024-09-21T10:25:48.755060+0000","last_unstale":"2024-09-21T10:25:54.807347+0000","last_undegraded":"2024-09-21T10:25:54.807347+0000","last_fullsized":"2024-09-21T10:25:54.807347+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:09.230030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":224,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892163+0000","last_change":"2024-09-21T10:25:48.754659+0000","last_active":"2024-09-21T10:26:07.892163+0000","last_peered":"2024-09-21T10:26:07.892163+0000","last_clean":"2024-09-21T10:26:07.892163+0000","last_became_active":"2024-09-21T10:25:48.754230+0000","last_became_peered":"2024-09-21T10:25:48.754230+0000","last_unstale":"2024-09-21T10:26:07.892163+0000","last_undegraded":"2024-09-21T10:26:07.892163+0000","last_fullsized":"2024-09-21T10:26:07.892163+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:12.555868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":31,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892065+0000","last_change":"2024-09-21T10:25:48.761249+0000","last_active":"2024-09-21T10:26:07.892065+0000","last_peered":"2024-09-21T10:26:07.892065+0000","last_clean":"2024-09-21T10:26:07.892065+0000","last_became_active":"2024-09-21T10:25:48.759359+0000","last_became_peered":"2024-09-21T10:25:48.759359+0000","last_unstale":"2024-09-21T10:26:07.892065+0000","last_undegraded":"2024-09-21T10:26:07.892065+0000","last_fullsized":"2024-09-21T10:26:07.892065+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:33.510936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"393'8319","reported_seq":8342,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:10.321151+0000","last_change":"2024-09-21T10:24:25.827798+0000","last_active":"2024-09-21T10:26:10.321151+0000","last_peered":"2024-09-21T10:26:10.321151+0000","last_clean":"2024-09-21T10:26:10.321151+0000","last_became_active":"2024-09-21T10:24:25.827230+0000","last_became_peered":"2024-09-21T10:24:25.827230+0000","last_unstale":"2024-09-21T10:26:10.321151+0000","last_undegraded":"2024-09-21T10:26:10.321151+0000","last_fullsized":"2024-09-21T10:26:10.321151+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8319,"log_dups_size":0,"ondisk_log_size":8319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:28:46.430403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":506789888,"num_objects":7733,"num_object_clones":0,"num_object_copies":23199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8319,"num_write_kb":513664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":693,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.807417+0000","last_change":"2024-09-21T10:25:48.755509+0000","last_active":"2024-09-21T10:25:54.807417+0000","last_peered":"2024-09-21T10:25:54.807417+0000","last_clean":"2024-09-21T10:25:54.807417+0000","last_became_active":"2024-09-21T10:25:48.754558+0000","last_became_peered":"2024-09-21T10:25:48.754558+0000","last_unstale":"2024-09-21T10:25:54.807417+0000","last_undegraded":"2024-09-21T10:25:54.807417+0000","last_fullsized":"2024-09-21T10:25:54.807417+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:11.262503+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1225,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.884846+0000","last_change":"2024-09-21T10:25:48.753359+0000","last_active":"2024-09-21T10:26:07.884846+0000","last_peered":"2024-09-21T10:26:07.884846+0000","last_clean":"2024-09-21T10:26:07.884846+0000","last_became_active":"2024-09-21T10:25:48.752653+0000","last_became_peered":"2024-09-21T10:25:48.752653+0000","last_unstale":"2024-09-21T10:26:07.884846+0000","last_undegraded":"2024-09-21T10:26:07.884846+0000","last_fullsized":"2024-09-21T10:26:07.884846+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:55.095131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":372,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892201+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:26:07.892201+0000","last_peered":"2024-09-21T10:26:07.892201+0000","last_clean":"2024-09-21T10:26:07.892201+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:26:07.892201+0000","last_undegraded":"2024-09-21T10:26:07.892201+0000","last_fullsized":"2024-09-21T10:26:07.892201+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":413,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.893501+0000","last_change":"2024-09-21T10:25:48.766755+0000","last_active":"2024-09-21T10:26:07.893501+0000","last_peered":"2024-09-21T10:26:07.893501+0000","last_clean":"2024-09-21T10:26:07.893501+0000","last_became_active":"2024-09-21T10:25:48.766298+0000","last_became_peered":"2024-09-21T10:25:48.766298+0000","last_unstale":"2024-09-21T10:26:07.893501+0000","last_undegraded":"2024-09-21T10:26:07.893501+0000","last_fullsized":"2024-09-21T10:26:07.893501+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:18.788447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"393'8406","reported_seq":8450,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:26:10.305863+0000","last_change":"2024-09-21T10:25:50.515832+0000","last_active":"2024-09-21T10:26:10.305863+0000","last_peered":"2024-09-21T10:26:10.305863+0000","last_clean":"2024-09-21T10:24:52.091731+0000","last_became_active":"2024-09-21T10:25:50.504214+0000","last_became_peered":"2024-09-21T10:25:50.504214+0000","last_unstale":"2024-09-21T10:26:10.305863+0000","last_undegraded":"2024-09-21T10:25:49.774199+0000","last_fullsized":"2024-09-21T10:25:49.743948+0000","mapping_epoch":390,"log_start":"382'400","ondisk_log_start":"382'400","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8006,"log_dups_size":0,"ondisk_log_size":8006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:49.426271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":506724352,"num_objects":7732,"num_object_clones":0,"num_object_copies":23196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8406,"num_write_kb":516416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":3046}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":643,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.805799+0000","last_change":"2024-09-21T10:25:48.757672+0000","last_active":"2024-09-21T10:25:54.805799+0000","last_peered":"2024-09-21T10:25:54.805799+0000","last_clean":"2024-09-21T10:25:54.805799+0000","last_became_active":"2024-09-21T10:25:48.757296+0000","last_became_peered":"2024-09-21T10:25:48.757296+0000","last_unstale":"2024-09-21T10:25:54.805799+0000","last_undegraded":"2024-09-21T10:25:54.805799+0000","last_fullsized":"2024-09-21T10:25:54.805799+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:31.244809+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:31.244809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:32.654430+0000","scrub_duration":0.0015828649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1136,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.805036+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:25:54.805036+0000","last_peered":"2024-09-21T10:25:54.805036+0000","last_clean":"2024-09-21T10:25:54.805036+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:25:54.805036+0000","last_undegraded":"2024-09-21T10:25:54.805036+0000","last_fullsized":"2024-09-21T10:25:54.805036+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":105,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:26:05.148329+0000","last_change":"2024-09-21T10:25:48.753400+0000","last_active":"2024-09-21T10:26:05.148329+0000","last_peered":"2024-09-21T10:26:05.148329+0000","last_clean":"2024-09-21T10:26:05.148329+0000","last_became_active":"2024-09-21T10:25:48.753088+0000","last_became_peered":"2024-09-21T10:25:48.753088+0000","last_unstale":"2024-09-21T10:26:05.148329+0000","last_undegraded":"2024-09-21T10:26:05.148329+0000","last_fullsized":"2024-09-21T10:26:05.148329+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:49.107566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":372,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892532+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:26:07.892532+0000","last_peered":"2024-09-21T10:26:07.892532+0000","last_clean":"2024-09-21T10:26:07.892532+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:26:07.892532+0000","last_undegraded":"2024-09-21T10:26:07.892532+0000","last_fullsized":"2024-09-21T10:26:07.892532+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"393'8328","reported_seq":8352,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:10.321251+0000","last_change":"2024-09-21T10:24:25.831730+0000","last_active":"2024-09-21T10:26:10.321251+0000","last_peered":"2024-09-21T10:26:10.321251+0000","last_clean":"2024-09-21T10:26:10.321251+0000","last_became_active":"2024-09-21T10:24:25.831416+0000","last_became_peered":"2024-09-21T10:24:25.831416+0000","last_unstale":"2024-09-21T10:26:10.321251+0000","last_undegraded":"2024-09-21T10:26:10.321251+0000","last_fullsized":"2024-09-21T10:26:10.321251+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8328,"log_dups_size":0,"ondisk_log_size":8328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.763992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":502267904,"num_objects":7664,"num_object_clones":0,"num_object_copies":22992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8328,"num_write_kb":511744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":476,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.812556+0000","last_change":"2024-09-21T10:24:24.828950+0000","last_active":"2024-09-21T10:25:54.812556+0000","last_peered":"2024-09-21T10:25:54.812556+0000","last_clean":"2024-09-21T10:25:54.812556+0000","last_became_active":"2024-09-21T10:24:24.828550+0000","last_became_peered":"2024-09-21T10:24:24.828550+0000","last_unstale":"2024-09-21T10:25:54.812556+0000","last_undegraded":"2024-09-21T10:25:54.812556+0000","last_fullsized":"2024-09-21T10:25:54.812556+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:29.313198+0000","scrub_duration":0.001370488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":341,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.805079+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:25:54.805079+0000","last_peered":"2024-09-21T10:25:54.805079+0000","last_clean":"2024-09-21T10:25:54.805079+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:25:54.805079+0000","last_undegraded":"2024-09-21T10:25:54.805079+0000","last_fullsized":"2024-09-21T10:25:54.805079+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":992,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892746+0000","last_change":"2024-09-21T10:25:48.760586+0000","last_active":"2024-09-21T10:26:07.892746+0000","last_peered":"2024-09-21T10:26:07.892746+0000","last_clean":"2024-09-21T10:26:07.892746+0000","last_became_active":"2024-09-21T10:25:48.760042+0000","last_became_peered":"2024-09-21T10:25:48.760042+0000","last_unstale":"2024-09-21T10:26:07.892746+0000","last_undegraded":"2024-09-21T10:26:07.892746+0000","last_fullsized":"2024-09-21T10:26:07.892746+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:41.526062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":408,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.894795+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:26:07.894795+0000","last_peered":"2024-09-21T10:26:07.894795+0000","last_clean":"2024-09-21T10:26:07.894795+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:26:07.894795+0000","last_undegraded":"2024-09-21T10:26:07.894795+0000","last_fullsized":"2024-09-21T10:26:07.894795+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"393'8265","reported_seq":8289,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:10.003392+0000","last_change":"2024-09-21T10:24:25.831693+0000","last_active":"2024-09-21T10:26:10.003392+0000","last_peered":"2024-09-21T10:26:10.003392+0000","last_clean":"2024-09-21T10:26:10.003392+0000","last_became_active":"2024-09-21T10:24:25.831245+0000","last_became_peered":"2024-09-21T10:24:25.831245+0000","last_unstale":"2024-09-21T10:26:10.003392+0000","last_undegraded":"2024-09-21T10:26:10.003392+0000","last_fullsized":"2024-09-21T10:26:10.003392+0000","mapping_epoch":378,"log_start":"382'1200","ondisk_log_start":"382'1200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7065,"log_dups_size":0,"ondisk_log_size":7065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:30.943590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":499449856,"num_objects":7621,"num_object_clones":0,"num_object_copies":22863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8265,"num_write_kb":508352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":491,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.812572+0000","last_change":"2024-09-21T10:25:48.752455+0000","last_active":"2024-09-21T10:25:54.812572+0000","last_peered":"2024-09-21T10:25:54.812572+0000","last_clean":"2024-09-21T10:25:54.812572+0000","last_became_active":"2024-09-21T10:25:48.752056+0000","last_became_peered":"2024-09-21T10:25:48.752056+0000","last_unstale":"2024-09-21T10:25:54.812572+0000","last_undegraded":"2024-09-21T10:25:54.812572+0000","last_fullsized":"2024-09-21T10:25:54.812572+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:15.886186+0000","scrub_duration":0.0018106349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":390,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892662+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:26:07.892662+0000","last_peered":"2024-09-21T10:26:07.892662+0000","last_clean":"2024-09-21T10:26:07.892662+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:26:07.892662+0000","last_undegraded":"2024-09-21T10:26:07.892662+0000","last_fullsized":"2024-09-21T10:26:07.892662+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":975,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.808073+0000","last_change":"2024-09-21T10:25:48.755967+0000","last_active":"2024-09-21T10:25:54.808073+0000","last_peered":"2024-09-21T10:25:54.808073+0000","last_clean":"2024-09-21T10:25:54.808073+0000","last_became_active":"2024-09-21T10:25:48.755037+0000","last_became_peered":"2024-09-21T10:25:48.755037+0000","last_unstale":"2024-09-21T10:25:54.808073+0000","last_undegraded":"2024-09-21T10:25:54.808073+0000","last_fullsized":"2024-09-21T10:25:54.808073+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:26.414898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":239,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.891720+0000","last_change":"2024-09-21T10:25:48.767902+0000","last_active":"2024-09-21T10:26:07.891720+0000","last_peered":"2024-09-21T10:26:07.891720+0000","last_clean":"2024-09-21T10:26:07.891720+0000","last_became_active":"2024-09-21T10:25:48.765443+0000","last_became_peered":"2024-09-21T10:25:48.765443+0000","last_unstale":"2024-09-21T10:26:07.891720+0000","last_undegraded":"2024-09-21T10:26:07.891720+0000","last_fullsized":"2024-09-21T10:26:07.891720+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:26.622825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"393'8326","reported_seq":8350,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:10.087141+0000","last_change":"2024-09-21T10:24:25.827432+0000","last_active":"2024-09-21T10:26:10.087141+0000","last_peered":"2024-09-21T10:26:10.087141+0000","last_clean":"2024-09-21T10:26:10.087141+0000","last_became_active":"2024-09-21T10:24:25.827047+0000","last_became_peered":"2024-09-21T10:24:25.827047+0000","last_unstale":"2024-09-21T10:26:10.087141+0000","last_undegraded":"2024-09-21T10:26:10.087141+0000","last_fullsized":"2024-09-21T10:26:10.087141+0000","mapping_epoch":378,"log_start":"382'1300","ondisk_log_start":"382'1300","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:52.646714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":505085976,"num_objects":7708,"num_object_clones":0,"num_object_copies":23124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7708,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8326,"num_write_kb":513025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":628,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819923+0000","last_change":"2024-09-21T10:24:26.255840+0000","last_active":"2024-09-21T10:25:54.819923+0000","last_peered":"2024-09-21T10:25:54.819923+0000","last_clean":"2024-09-21T10:25:54.819923+0000","last_became_active":"2024-09-21T10:24:24.825549+0000","last_became_peered":"2024-09-21T10:24:24.825549+0000","last_unstale":"2024-09-21T10:25:54.819923+0000","last_undegraded":"2024-09-21T10:25:54.819923+0000","last_fullsized":"2024-09-21T10:25:54.819923+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:26.255753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.255753+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:25.638856+0000","scrub_duration":0.0016223839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":709,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.889880+0000","last_change":"2024-09-21T10:25:48.758590+0000","last_active":"2024-09-21T10:26:07.889880+0000","last_peered":"2024-09-21T10:26:07.889880+0000","last_clean":"2024-09-21T10:26:07.889880+0000","last_became_active":"2024-09-21T10:25:48.758189+0000","last_became_peered":"2024-09-21T10:25:48.758189+0000","last_unstale":"2024-09-21T10:26:07.889880+0000","last_undegraded":"2024-09-21T10:26:07.889880+0000","last_fullsized":"2024-09-21T10:26:07.889880+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:24.126237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1043,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.894108+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:26:07.894108+0000","last_peered":"2024-09-21T10:26:07.894108+0000","last_clean":"2024-09-21T10:26:07.894108+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:26:07.894108+0000","last_undegraded":"2024-09-21T10:26:07.894108+0000","last_fullsized":"2024-09-21T10:26:07.894108+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":705,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.885587+0000","last_change":"2024-09-21T10:25:48.754966+0000","last_active":"2024-09-21T10:26:07.885587+0000","last_peered":"2024-09-21T10:26:07.885587+0000","last_clean":"2024-09-21T10:26:07.885587+0000","last_became_active":"2024-09-21T10:25:48.754152+0000","last_became_peered":"2024-09-21T10:25:48.754152+0000","last_unstale":"2024-09-21T10:26:07.885587+0000","last_undegraded":"2024-09-21T10:26:07.885587+0000","last_fullsized":"2024-09-21T10:26:07.885587+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:33:28.795713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"393'8500","reported_seq":8541,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:14.144558+0000","last_change":"2024-09-21T10:24:25.830908+0000","last_active":"2024-09-21T10:26:14.144558+0000","last_peered":"2024-09-21T10:26:14.144558+0000","last_clean":"2024-09-21T10:26:14.144558+0000","last_became_active":"2024-09-21T10:24:25.830171+0000","last_became_peered":"2024-09-21T10:24:25.830171+0000","last_unstale":"2024-09-21T10:26:14.144558+0000","last_undegraded":"2024-09-21T10:26:14.144558+0000","last_fullsized":"2024-09-21T10:26:14.144558+0000","mapping_epoch":378,"log_start":"382'1600","ondisk_log_start":"382'1600","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6900,"log_dups_size":0,"ondisk_log_size":6900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:00.292732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":487981056,"num_objects":7446,"num_object_clones":0,"num_object_copies":22338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8500,"num_write_kb":510272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":676,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.883789+0000","last_change":"2024-09-21T10:24:26.926717+0000","last_active":"2024-09-21T10:26:07.883789+0000","last_peered":"2024-09-21T10:26:07.883789+0000","last_clean":"2024-09-21T10:26:07.883789+0000","last_became_active":"2024-09-21T10:24:24.828205+0000","last_became_peered":"2024-09-21T10:24:24.828205+0000","last_unstale":"2024-09-21T10:26:07.883789+0000","last_undegraded":"2024-09-21T10:26:07.883789+0000","last_fullsized":"2024-09-21T10:26:07.883789+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:26.926661+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.926661+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:25.847900+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":60,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892371+0000","last_change":"2024-09-21T10:25:48.762565+0000","last_active":"2024-09-21T10:26:07.892371+0000","last_peered":"2024-09-21T10:26:07.892371+0000","last_clean":"2024-09-21T10:26:07.892371+0000","last_became_active":"2024-09-21T10:25:48.761324+0000","last_became_peered":"2024-09-21T10:25:48.761324+0000","last_unstale":"2024-09-21T10:26:07.892371+0000","last_undegraded":"2024-09-21T10:26:07.892371+0000","last_fullsized":"2024-09-21T10:26:07.892371+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:58.885790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":704,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:26:05.067291+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:26:05.067291+0000","last_peered":"2024-09-21T10:26:05.067291+0000","last_clean":"2024-09-21T10:26:05.067291+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:26:05.067291+0000","last_undegraded":"2024-09-21T10:26:05.067291+0000","last_fullsized":"2024-09-21T10:26:05.067291+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":366,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.886878+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:26:07.886878+0000","last_peered":"2024-09-21T10:26:07.886878+0000","last_clean":"2024-09-21T10:26:07.886878+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:26:07.886878+0000","last_undegraded":"2024-09-21T10:26:07.886878+0000","last_fullsized":"2024-09-21T10:26:07.886878+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":390,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819690+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:25:54.819690+0000","last_peered":"2024-09-21T10:25:54.819690+0000","last_clean":"2024-09-21T10:25:54.819690+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:25:54.819690+0000","last_undegraded":"2024-09-21T10:25:54.819690+0000","last_fullsized":"2024-09-21T10:25:54.819690+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":99,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819773+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:25:54.819773+0000","last_peered":"2024-09-21T10:25:54.819773+0000","last_clean":"2024-09-21T10:25:54.819773+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:25:54.819773+0000","last_undegraded":"2024-09-21T10:25:54.819773+0000","last_fullsized":"2024-09-21T10:25:54.819773+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":93,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.804954+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:25:54.804954+0000","last_peered":"2024-09-21T10:25:54.804954+0000","last_clean":"2024-09-21T10:25:54.804954+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:25:54.804954+0000","last_undegraded":"2024-09-21T10:25:54.804954+0000","last_fullsized":"2024-09-21T10:25:54.804954+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":72,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.807802+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:25:54.807802+0000","last_peered":"2024-09-21T10:25:54.807802+0000","last_clean":"2024-09-21T10:25:54.807802+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:25:54.807802+0000","last_undegraded":"2024-09-21T10:25:54.807802+0000","last_fullsized":"2024-09-21T10:25:54.807802+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":101,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.805269+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:25:54.805269+0000","last_peered":"2024-09-21T10:25:54.805269+0000","last_clean":"2024-09-21T10:25:54.805269+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:25:54.805269+0000","last_undegraded":"2024-09-21T10:25:54.805269+0000","last_fullsized":"2024-09-21T10:25:54.805269+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":617,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.881615+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:26:07.881615+0000","last_peered":"2024-09-21T10:26:07.881615+0000","last_clean":"2024-09-21T10:26:07.881615+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:26:07.881615+0000","last_undegraded":"2024-09-21T10:26:07.881615+0000","last_fullsized":"2024-09-21T10:26:07.881615+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":237,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892733+0000","last_change":"2024-09-21T10:25:48.765786+0000","last_active":"2024-09-21T10:26:07.892733+0000","last_peered":"2024-09-21T10:26:07.892733+0000","last_clean":"2024-09-21T10:26:07.892733+0000","last_became_active":"2024-09-21T10:25:48.764011+0000","last_became_peered":"2024-09-21T10:25:48.764011+0000","last_unstale":"2024-09-21T10:26:07.892733+0000","last_undegraded":"2024-09-21T10:26:07.892733+0000","last_fullsized":"2024-09-21T10:26:07.892733+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:13.352641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":677,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.885511+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:26:07.885511+0000","last_peered":"2024-09-21T10:26:07.885511+0000","last_clean":"2024-09-21T10:26:07.885511+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:26:07.885511+0000","last_undegraded":"2024-09-21T10:26:07.885511+0000","last_fullsized":"2024-09-21T10:26:07.885511+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":408,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.812626+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:25:54.812626+0000","last_peered":"2024-09-21T10:25:54.812626+0000","last_clean":"2024-09-21T10:25:54.812626+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:25:54.812626+0000","last_undegraded":"2024-09-21T10:25:54.812626+0000","last_fullsized":"2024-09-21T10:25:54.812626+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":705,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.886617+0000","last_change":"2024-09-21T10:25:48.755341+0000","last_active":"2024-09-21T10:26:07.886617+0000","last_peered":"2024-09-21T10:26:07.886617+0000","last_clean":"2024-09-21T10:26:07.886617+0000","last_became_active":"2024-09-21T10:25:48.754620+0000","last_became_peered":"2024-09-21T10:25:48.754620+0000","last_unstale":"2024-09-21T10:26:07.886617+0000","last_undegraded":"2024-09-21T10:26:07.886617+0000","last_fullsized":"2024-09-21T10:26:07.886617+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:57:21.296549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":270,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892164+0000","last_change":"2024-09-21T10:25:48.765683+0000","last_active":"2024-09-21T10:26:07.892164+0000","last_peered":"2024-09-21T10:26:07.892164+0000","last_clean":"2024-09-21T10:26:07.892164+0000","last_became_active":"2024-09-21T10:25:48.765281+0000","last_became_peered":"2024-09-21T10:25:48.765281+0000","last_unstale":"2024-09-21T10:26:07.892164+0000","last_undegraded":"2024-09-21T10:26:07.892164+0000","last_fullsized":"2024-09-21T10:26:07.892164+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:41.798683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":392,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.893825+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:26:07.893825+0000","last_peered":"2024-09-21T10:26:07.893825+0000","last_clean":"2024-09-21T10:26:07.893825+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:26:07.893825+0000","last_undegraded":"2024-09-21T10:26:07.893825+0000","last_fullsized":"2024-09-21T10:26:07.893825+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":683,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.882839+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:26:07.882839+0000","last_peered":"2024-09-21T10:26:07.882839+0000","last_clean":"2024-09-21T10:26:07.882839+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:26:07.882839+0000","last_undegraded":"2024-09-21T10:26:07.882839+0000","last_fullsized":"2024-09-21T10:26:07.882839+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":397,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819504+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:25:54.819504+0000","last_peered":"2024-09-21T10:25:54.819504+0000","last_clean":"2024-09-21T10:25:54.819504+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:25:54.819504+0000","last_undegraded":"2024-09-21T10:25:54.819504+0000","last_fullsized":"2024-09-21T10:25:54.819504+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":683,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.885171+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:26:07.885171+0000","last_peered":"2024-09-21T10:26:07.885171+0000","last_clean":"2024-09-21T10:26:07.885171+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:26:07.885171+0000","last_undegraded":"2024-09-21T10:26:07.885171+0000","last_fullsized":"2024-09-21T10:26:07.885171+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":30,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.893184+0000","last_change":"2024-09-21T10:25:48.767385+0000","last_active":"2024-09-21T10:26:07.893184+0000","last_peered":"2024-09-21T10:26:07.893184+0000","last_clean":"2024-09-21T10:26:07.893184+0000","last_became_active":"2024-09-21T10:25:48.764077+0000","last_became_peered":"2024-09-21T10:25:48.764077+0000","last_unstale":"2024-09-21T10:26:07.893184+0000","last_undegraded":"2024-09-21T10:26:07.893184+0000","last_fullsized":"2024-09-21T10:26:07.893184+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:06:51.969987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819609+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:25:54.819609+0000","last_peered":"2024-09-21T10:25:54.819609+0000","last_clean":"2024-09-21T10:25:54.819609+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:25:54.819609+0000","last_undegraded":"2024-09-21T10:25:54.819609+0000","last_fullsized":"2024-09-21T10:25:54.819609+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":333,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.804711+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:25:54.804711+0000","last_peered":"2024-09-21T10:25:54.804711+0000","last_clean":"2024-09-21T10:25:54.804711+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:25:54.804711+0000","last_undegraded":"2024-09-21T10:25:54.804711+0000","last_fullsized":"2024-09-21T10:25:54.804711+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":104,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819401+0000","last_change":"2024-09-21T10:25:48.753233+0000","last_active":"2024-09-21T10:25:54.819401+0000","last_peered":"2024-09-21T10:25:54.819401+0000","last_clean":"2024-09-21T10:25:54.819401+0000","last_became_active":"2024-09-21T10:25:48.752932+0000","last_became_peered":"2024-09-21T10:25:48.752932+0000","last_unstale":"2024-09-21T10:25:54.819401+0000","last_undegraded":"2024-09-21T10:25:54.819401+0000","last_fullsized":"2024-09-21T10:25:54.819401+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:49.456475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":311,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.805303+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:25:54.805303+0000","last_peered":"2024-09-21T10:25:54.805303+0000","last_clean":"2024-09-21T10:25:54.805303+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:25:54.805303+0000","last_undegraded":"2024-09-21T10:25:54.805303+0000","last_fullsized":"2024-09-21T10:25:54.805303+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":708,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.881898+0000","last_change":"2024-09-21T10:25:48.755262+0000","last_active":"2024-09-21T10:26:07.881898+0000","last_peered":"2024-09-21T10:26:07.881898+0000","last_clean":"2024-09-21T10:26:07.881898+0000","last_became_active":"2024-09-21T10:25:48.754746+0000","last_became_peered":"2024-09-21T10:25:48.754746+0000","last_unstale":"2024-09-21T10:26:07.881898+0000","last_undegraded":"2024-09-21T10:26:07.881898+0000","last_fullsized":"2024-09-21T10:26:07.881898+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:28:17.312418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":659,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.883225+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:26:07.883225+0000","last_peered":"2024-09-21T10:26:07.883225+0000","last_clean":"2024-09-21T10:26:07.883225+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:26:07.883225+0000","last_undegraded":"2024-09-21T10:26:07.883225+0000","last_fullsized":"2024-09-21T10:26:07.883225+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-09-21T10:25:54.819247+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:25:54.819247+0000","last_peered":"2024-09-21T10:25:54.819247+0000","last_clean":"2024-09-21T10:25:54.819247+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:25:54.819247+0000","last_undegraded":"2024-09-21T10:25:54.819247+0000","last_fullsized":"2024-09-21T10:25:54.819247+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":62,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-09-21T10:26:07.892490+0000","last_change":"2024-09-21T10:25:48.764108+0000","last_active":"2024-09-21T10:26:07.892490+0000","last_peered":"2024-09-21T10:26:07.892490+0000","last_clean":"2024-09-21T10:26:07.892490+0000","last_became_active":"2024-09-21T10:25:48.761361+0000","last_became_peered":"2024-09-21T10:25:48.761361+0000","last_unstale":"2024-09-21T10:26:07.892490+0000","last_undegraded":"2024-09-21T10:26:07.892490+0000","last_fullsized":"2024-09-21T10:26:07.892490+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:49.188529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":7985561624,"num_objects":121851,"num_object_clones":0,"num_object_copies":365553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17752,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":121851,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133749,"num_write_kb":8179137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":630,"num_bytes_recovered":41287680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":22789369856,"data_stored":22789357640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120949,"ondisk_log_size":120949,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28818,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234438,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5231460,"kb_used_data":4464476,"kb_used_omap":1,"kb_used_meta":766974,"kb_avail":88513692,"statfs":{"total":95995035648,"available":90638020608,"internally_reserved":0,"allocated":4571623424,"data_stored":4569591954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":785381835},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496076,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2181532,"kb_used_data":1514644,"kb_used_omap":1,"kb_used_meta":666878,"kb_avail":91563620,"statfs":{"total":95995035648,"available":93761146880,"internally_reserved":0,"allocated":1550995456,"data_stored":1548972358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":682883527},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403976,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4886496,"kb_used_data":4429524,"kb_used_omap":7,"kb_used_meta":456888,"kb_avail":88858656,"statfs":{"total":95995035648,"available":90991263744,"internally_reserved":0,"allocated":4535832576,"data_stored":4533805872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":467854232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019349,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2110308,"kb_used_data":1483608,"kb_used_omap":1,"kb_used_meta":626686,"kb_avail":91634844,"statfs":{"total":95995035648,"available":93834080256,"internally_reserved":0,"allocated":1519214592,"data_stored":1517187455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":641726918},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435534,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4716368,"kb_used_data":3840456,"kb_used_omap":5,"kb_used_meta":875770,"kb_avail":89028784,"statfs":{"total":95995035648,"available":91165474816,"internally_reserved":0,"allocated":3932626944,"data_stored":3930611935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":896788520},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599061,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2199184,"kb_used_data":1437256,"kb_used_omap":7,"kb_used_meta":761912,"kb_avail":91545968,"statfs":{"total":95995035648,"available":93743071232,"internally_reserved":0,"allocated":1471750144,"data_stored":1469735353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":780198349},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762587,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5127628,"kb_used_data":4419844,"kb_used_omap":7,"kb_used_meta":707768,"kb_avail":88617524,"statfs":{"total":95995035648,"available":90744344576,"internally_reserved":0,"allocated":4525920256,"data_stored":4523908479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":724754685},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":389,"seq":1670742278150,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":878800,"kb_used_data":818632,"kb_used_omap":12,"kb_used_meta":60147,"kb_avail":92866352,"statfs":{"total":95995035648,"available":95095144448,"internally_reserved":0,"allocated":838279168,"data_stored":836264129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12595,"internal_metadata":61591245},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11001,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":824967168,"data_stored":824967168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4513075200,"data_stored":4513071128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1459093504,"data_stored":1459093504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3917807616,"data_stored":3917807616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1506082816,"data_stored":1506082816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4515434496,"data_stored":4515430424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1510801408,"data_stored":1510801408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4542107648,"data_stored":4542103576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:26:16.287 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 11 2024-09-21T10:26:16.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:16.382 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:16.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:16.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:16 smithi032 ceph-mon[101299]: pgmap v1204: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 780 op/s; 17752/365637 objects degraded (4.855%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:16.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:16 smithi032 ceph-mon[101299]: from='client.45917 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:26:16.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:16 smithi032 ceph-mon[105748]: pgmap v1204: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 780 op/s; 17752/365637 objects degraded (4.855%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:16.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:16 smithi032 ceph-mon[105748]: from='client.45917 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:26:16.481 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:16 smithi045 ceph-mon[95976]: pgmap v1204: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 780 op/s; 17752/365637 objects degraded (4.855%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:16.481 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:16 smithi045 ceph-mon[95976]: from='client.45917 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:26:16.484 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:16.522 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:26:16.528 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:16.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:16.631 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:16.684 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:16.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:16.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:16.787 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:16.834 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:16.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:16.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:16.971 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:16.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:16.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:17.022 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:17.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:17.078 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:17.094 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:26:17.094 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:26:17.112 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:17.119 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:17.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:17.128 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:26:17.128 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:26:17.161 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:26:17.162 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:26:17.202 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:26:17.203 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:26:17.221 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:17.234 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:26:17.234 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:26:17.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:17.264 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:17.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:17.276 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:26:17.276 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:26:17.312 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:26:17.312 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:26:17.348 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:26:17.348 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:26:17.364 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:17.386 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 17752/365637 objects degraded (4.855%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:17.386 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:17 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2603033722' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:17.386 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 17752/365637 objects degraded (4.855%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:17.386 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:17 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2603033722' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:17.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:17.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:17.456 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 17752/365637 objects degraded (4.855%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:17.456 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:17 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2603033722' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:17.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:17.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:17.547 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:17.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:17.648 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:17.681 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:17.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:17.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:17.782 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:17.826 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:17.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:17.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:17.961 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:17.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:17.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:18.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:18.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:18.098 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:18.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:18.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:18.242 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:18.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:18.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:18.345 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:18.398 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:18.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:18 smithi032 ceph-mon[105748]: pgmap v1205: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 17595/356145 objects degraded (4.940%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:26:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:18 smithi032 ceph-mon[101299]: pgmap v1205: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 17595/356145 objects degraded (4.940%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:26:18.500 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:18 smithi045 ceph-mon[95976]: pgmap v1205: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 17595/356145 objects degraded (4.940%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:26:18.537 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:18.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:18.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:18.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:18.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:18.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:18.767 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:18.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:18.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:18.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:18.942 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:18.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:19.043 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:19.079 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:19.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:19.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:19.180 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:19.224 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:19.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:19.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:19.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:19.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:19.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:19.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:19.464 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:19.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:19.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:19.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:19.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:19.638 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:19.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:19.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:19.740 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:19.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:19.771 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:19.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:19.872 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:19.910 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:19.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:19.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:19.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:26:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:26:19] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-21T10:26:20.012 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:20.054 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:20.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:20.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:20.156 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:20.194 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:20.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:20.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:20.260 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:20 smithi032 ceph-mon[105748]: pgmap v1206: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 17595/356145 objects degraded (4.940%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:26:20.260 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:20 smithi032 ceph-mon[101299]: pgmap v1206: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 17595/356145 objects degraded (4.940%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:26:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:20 smithi045 ceph-mon[95976]: pgmap v1206: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 17595/356145 objects degraded (4.940%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:26:20.296 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:20.335 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:20.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:20.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:20.437 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:20.466 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:20.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:20.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:20.570 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:20.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:20.615 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:20.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:20.716 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:20.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:20.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:20.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:20.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:20.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:20.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:20.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:21.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:21.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:21.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:21.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:21.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:21.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:21.234 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 1 pgp_num to 11 2024-09-21T10:26:21.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:21.308 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:21.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:21.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:21.409 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:21.453 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:21.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:21.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:21.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:21 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1932773751' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:26:21.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:21 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1932773751' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:26:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:21 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1932773751' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:26:21.555 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:21.597 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:21.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:21.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:21.700 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:21.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:21.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:21.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:21.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:21.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:21.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:21.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:21.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:22.032 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:22.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:22.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:26:22.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:22.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:22.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:22.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:22.295 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:22.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:22.397 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:22.431 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:22.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:22 smithi032 ceph-mon[105748]: pgmap v1207: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 980 op/s; 17469/346209 objects degraded (5.046%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:22 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1932773751' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T10:26:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:22 smithi032 ceph-mon[105748]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T10:26:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 17469/346209 objects degraded (5.046%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:22 smithi032 ceph-mon[101299]: pgmap v1207: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 980 op/s; 17469/346209 objects degraded (5.046%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:22 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1932773751' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T10:26:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:22 smithi032 ceph-mon[101299]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T10:26:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 17469/346209 objects degraded (5.046%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:22.533 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:22 smithi045 ceph-mon[95976]: pgmap v1207: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 980 op/s; 17469/346209 objects degraded (5.046%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:22.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:22 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1932773751' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T10:26:22.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:22 smithi045 ceph-mon[95976]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T10:26:22.536 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 17469/346209 objects degraded (5.046%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:22.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:22.561 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:22.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:22.661 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:22.686 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:22.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:22.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:22.788 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:22.813 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:22.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:22.915 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:22.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:22.950 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:22.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:23.051 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:23.090 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:23.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:23.192 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:23.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:23.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:23.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:23.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:23.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:23.372 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:23.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:23.473 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:23.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:23.521 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:23.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:23.622 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:23.663 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:23.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:23.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:23.764 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:23.799 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:23.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:23.901 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:23.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:23.942 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:23.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:24.043 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:24.083 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:24.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:24.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:24.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:24.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:24.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:24.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:24.373 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:24.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:24.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:24.435 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:24.475 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:24.503 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:24.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:24.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:24 smithi045 ceph-mon[95976]: pgmap v1209: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 17469/344784 objects degraded (5.067%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:26:24.581 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:24 smithi032 ceph-mon[101299]: pgmap v1209: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 17469/344784 objects degraded (5.067%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:26:24.581 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:24 smithi032 ceph-mon[105748]: pgmap v1209: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 17469/344784 objects degraded (5.067%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:26:24.605 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:24.652 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:24.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:24.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:24.754 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:24.789 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:24.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:24.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:24.900 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:26:24.900 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:26:24.900 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:26:24.900 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:26:24.900 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:26:24.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:24.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:25.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:25.069 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:25.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:25.171 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:25.211 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:25.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:25.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:25.256 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:26:25.313 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:25.348 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:25.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:25.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:25.450 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:25.491 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:25.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:25.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:25.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3267183997' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:26:25.593 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:25.621 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:26:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:25.636 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:25.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:25.670 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3267183997' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:26:25.670 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3267183997' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:26:25.737 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:25.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:25.776 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:25.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:25.877 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:25.928 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:25.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:25.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:26.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:26.052 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:26.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:26.067 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:26.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:26.168 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:26.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:26.218 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:26.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:26.319 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:26.358 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:26.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:26.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:26.460 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:26.469 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:26 smithi045 ceph-mon[95976]: pgmap v1210: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 17469/339744 objects degraded (5.142%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:26:26.469 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:26 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:26:26.469 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3267183997' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:26:26.469 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:26 smithi045 ceph-mon[95976]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T10:26:26.494 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:26.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:26.540 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:26 smithi032 ceph-mon[101299]: pgmap v1210: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 17469/339744 objects degraded (5.142%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:26:26.541 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:26 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:26:26.541 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3267183997' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:26:26.541 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:26 smithi032 ceph-mon[101299]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T10:26:26.547 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:26 smithi032 ceph-mon[105748]: pgmap v1210: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 17469/339744 objects degraded (5.142%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:26:26.547 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:26 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:26:26.547 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3267183997' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:26:26.547 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:26 smithi032 ceph-mon[105748]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T10:26:26.597 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:26.627 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:26.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:26.729 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:26.768 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:26.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:26.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:26.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:26.914 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:26.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:26.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:27.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:27.054 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:27.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:27.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:27.156 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:27.190 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:27.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:27.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:27.281 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:26:27.281 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:26:27.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:27.321 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:26:27.322 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:26:27.327 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:27.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:27.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:27.360 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:26:27.360 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:26:27.360 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:27 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 17469/339744 objects degraded (5.142%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:27.360 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3682900036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:27.361 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:27 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 17469/339744 objects degraded (5.142%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:27.361 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3682900036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:27.403 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:26:27.403 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:26:27.428 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:27.438 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:26:27.438 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:26:27.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:27.483 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:27.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:27.484 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:26:27.484 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:26:27.519 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:26:27.519 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:26:27.530 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:27 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 17469/339744 objects degraded (5.142%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:27.530 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3682900036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:27.553 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:26:27.554 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:26:27.584 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:27.621 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:27.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:27.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:27.768 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:27.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:27.869 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:27.914 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:27.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:28.016 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:28.026 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:26:28.058 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:28.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:28.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:28.096 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:26:28.096 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:26:28.161 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:28.202 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:28.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:28.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:28.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:28.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:28.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:28.436 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 1, 0, 4] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 6, 7, 5, 0] 2024-09-21T10:26:28.436 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:26:28.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:28.474 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:28.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:28.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:28 smithi045 ceph-mon[95976]: pgmap v1212: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 17300/333243 objects degraded (5.191%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:28.569 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:28 smithi032 ceph-mon[105748]: pgmap v1212: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 17300/333243 objects degraded (5.191%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:28.569 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:28 smithi032 ceph-mon[101299]: pgmap v1212: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 17300/333243 objects degraded (5.191%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:26:28.576 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:28.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:28.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:28.705 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:28.715 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:28.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:28.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:28.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:28.790 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:26:28.842 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:28.888 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:28.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:28.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:28.990 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:29.052 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:29.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:29.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:29.176 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:29.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:29.198 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:29.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:29.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:29.343 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:29.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:29.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:29.445 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:29.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:29 smithi032 ceph-mon[101299]: pgmap v1213: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 17300/333243 objects degraded (5.191%); 1.3 MiB/s, 21 objects/s recovering 2024-09-21T10:26:29.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:29 smithi032 ceph-mon[105748]: pgmap v1213: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 17300/333243 objects degraded (5.191%); 1.3 MiB/s, 21 objects/s recovering 2024-09-21T10:26:29.481 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:29.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:29.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:29.499 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:29.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:29 smithi045 ceph-mon[95976]: pgmap v1213: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 17300/333243 objects degraded (5.191%); 1.3 MiB/s, 21 objects/s recovering 2024-09-21T10:26:29.582 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:29.616 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:29.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:29.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:29.718 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:29.762 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:29.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:29.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:29.864 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:29.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:29.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:29.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:29.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:26:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:26:29] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-21T10:26:30.005 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:30.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:30.052 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:30.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:30.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:30.189 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:30.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:30.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:30.290 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:30.326 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:30.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:30.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:30.428 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:30.466 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:30.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:30.568 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:30.607 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:26:30.608 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:30.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:30.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:30.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:30.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:30.752 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:30.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:30.853 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:30.886 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:30.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:30.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:30.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:31.035 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:31.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:31.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:31.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:31.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:31.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:31.269 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:31.317 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:31.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:31.419 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:31.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:31.459 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:31.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:31.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:31.618 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:31.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:31.719 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:31.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:31.760 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:31.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:31.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:31 smithi045 ceph-mon[95976]: pgmap v1214: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 17149/322458 objects degraded (5.318%); 2.1 MiB/s, 34 objects/s recovering 2024-09-21T10:26:31.861 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:31.899 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:31.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:31.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:31.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:31 smithi032 ceph-mon[105748]: pgmap v1214: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 17149/322458 objects degraded (5.318%); 2.1 MiB/s, 34 objects/s recovering 2024-09-21T10:26:31.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:31 smithi032 ceph-mon[101299]: pgmap v1214: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 17149/322458 objects degraded (5.318%); 2.1 MiB/s, 34 objects/s recovering 2024-09-21T10:26:32.001 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:32.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:32.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:32.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:32.070 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:32.120 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:26:32.120 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:26:32.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:32.191 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:32.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:32.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:32.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:32.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:32.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:32.439 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:32.468 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:32.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:32.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:32.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:32.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:32.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:32.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:32.697 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:26:32.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:32.751 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:32.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:32.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:32.807 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:26:32.853 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:32.899 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:32.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:32 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1579457301' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:26:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:32 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:26:32.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:32 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 17149/322458 objects degraded (5.318%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:32.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:32 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1579457301' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:26:32.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:32 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:26:32.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:32 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 17149/322458 objects degraded (5.318%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:33.000 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:33.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:32 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1579457301' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:26:33.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:32 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:26:33.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:32 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 17149/322458 objects degraded (5.318%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:33.044 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:33.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:33.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:33.117 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:33.145 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:33.179 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:33.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:33.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:33.280 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:33.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:33.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:33.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:33.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:33.458 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:33.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:33.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:33.560 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:33.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:33.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:33.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:33.694 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:26:33.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:33.764 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:33.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:33.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:33.865 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:33.894 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:33.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[105748]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 17149/320814 objects degraded (5.345%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:26:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:26:33.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[105748]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T10:26:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/611615645' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:33.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:33.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[101299]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 17149/320814 objects degraded (5.345%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:26:33.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:33.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:26:33.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[101299]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T10:26:33.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/611615645' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:33.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:33.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:33.962 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:33 smithi045 ceph-mon[95976]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 17149/320814 objects degraded (5.345%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:26:33.962 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:33 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:33.962 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:33 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:26:33.962 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:33 smithi045 ceph-mon[95976]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T10:26:33.962 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:33 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/611615645' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:33.962 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:33 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:33.962 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:33.997 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:34.044 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:34.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:34.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:34.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:34.184 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:34.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:34.285 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:34.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:34.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:34.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:34.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:34.466 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:34.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:34.567 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:34.619 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:34.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:34.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:34.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:34.756 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:34.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:34.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:34.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:34 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:26:34.884 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:34 smithi032 ceph-mon[101299]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T10:26:34.884 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:34 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:26:34.885 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:34 smithi032 ceph-mon[105748]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T10:26:34.901 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:34.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:34.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:35.003 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:35.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:34 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:26:35.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:34 smithi045 ceph-mon[95976]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T10:26:35.042 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:35.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:35.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:35.093 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:26:35.093 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:26:35.144 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:35.179 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:35.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:35.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:35.281 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:35.337 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:35.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:35.437 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:35.474 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:35.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:35.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:35.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:35.615 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:35.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:35.718 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:35.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:35.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:35.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:35.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:35.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:35.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:35.912 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:26:35.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:35 smithi032 ceph-mon[105748]: pgmap v1218: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 17149/315693 objects degraded (5.432%); 1.2 MiB/s, 18 objects/s recovering 2024-09-21T10:26:35.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:35 smithi032 ceph-mon[101299]: pgmap v1218: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 17149/315693 objects degraded (5.432%); 1.2 MiB/s, 18 objects/s recovering 2024-09-21T10:26:35.989 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:35 smithi045 ceph-mon[95976]: pgmap v1218: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 17149/315693 objects degraded (5.432%); 1.2 MiB/s, 18 objects/s recovering 2024-09-21T10:26:35.990 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:36.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:36.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:36.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:26:36.133 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:36.182 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:36.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:36.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:36.284 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:36.311 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:36.329 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:36.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:36.431 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:36.469 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:36.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:36.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:36.613 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:36.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:36.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:36.715 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:36.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:36.781 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:36.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:36.790 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:36.813 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:26:36.813 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:26:36.863 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:26:36.863 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:26:36.883 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:36.896 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:26:36.896 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:26:36.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:36.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:36.919 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:26:36.920 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:26:36.939 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:26:36.939 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:26:36.970 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:26:36.971 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:26:37.005 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:26:37.005 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:26:37.018 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:37.040 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:26:37.040 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:26:37.061 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:37.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:37.163 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:37.203 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:37.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:37.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:37.304 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:37.340 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:37.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:37.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:37.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:37.485 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:37.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:37.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:37.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:37.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:37.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:37.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:37.663 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:26:37.725 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:37.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:37 smithi045 ceph-mon[95976]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 16984/309195 objects degraded (5.493%); 2.5 MiB/s, 39 objects/s recovering 2024-09-21T10:26:37.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:37 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/607911774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:37.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:37 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16984/309195 objects degraded (5.493%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:37.789 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:37.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:37.891 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:37.919 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:37.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:37.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:37.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:37 smithi032 ceph-mon[101299]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 16984/309195 objects degraded (5.493%); 2.5 MiB/s, 39 objects/s recovering 2024-09-21T10:26:37.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:37 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/607911774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:37.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:37 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16984/309195 objects degraded (5.493%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:37.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:37 smithi032 ceph-mon[105748]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 16984/309195 objects degraded (5.493%); 2.5 MiB/s, 39 objects/s recovering 2024-09-21T10:26:37.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:37 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/607911774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:37.959 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:37 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16984/309195 objects degraded (5.493%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:38.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:38.070 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:38.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:38.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:38.207 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:38.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:38.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:38.308 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:38.341 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:38.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:38.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:38.443 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:38.484 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:38.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:38.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:38.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:38.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:38.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:38.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:38.723 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:38.890 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:38.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:38.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:38.991 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:39.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:39.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:39.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:39.160 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:39.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:39.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:39.262 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:39.293 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:39.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:39.395 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:39.434 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:39.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:39.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:39.535 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:39.569 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:39.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:39.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:39.671 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:39.711 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:39.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:39.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:39.813 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:39.852 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:39.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:26:39.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:39.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:39.932 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:39 smithi032 ceph-mon[101299]: pgmap v1220: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 16984/309195 objects degraded (5.493%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:26:39.932 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:26:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:26:39] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.51.0" 2024-09-21T10:26:39.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:39 smithi032 ceph-mon[105748]: pgmap v1220: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 16984/309195 objects degraded (5.493%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:26:39.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:39.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:39.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:39 smithi045 ceph-mon[95976]: pgmap v1220: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 16984/309195 objects degraded (5.493%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:26:40.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:40.123 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:40.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:40.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:40.224 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:40.253 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:40.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:40.298 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:40.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:40.399 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:40.434 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:40.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:40.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:40.467 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:40.536 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:40.577 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:40.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:40.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:40.719 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:40.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:40.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:40.821 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:40.858 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:40.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:40.959 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:40.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:40.992 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:40.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:41.093 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:41.133 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:41.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:41.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:41.210 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:26:41.210 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:26:41.210 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:200300 7.d 7:bd9bdcaa:::benchmark_data_smithi032_200324_object75634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:48.592311+0000", 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.61270923100005, 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0679320990000001, 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 200300 2024-09-21T10:26:41.211 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.592311+0000", 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.592311+0000", 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.212 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.592309+0000", 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.592315+0000", 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6160000000000001e-06 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.592325+0000", 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0404999999999999e-05 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.213 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.592331+0000", 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4369999999999998e-06 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.592390+0000", 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9129999999999998e-05 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.592401+0000", 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0808000000000001e-05 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.214 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.603111+0000", 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.010709793 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.603195+0000", 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4470999999999998e-05 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.603264+0000", 2024-09-21T10:26:41.215 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9333999999999997e-05 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.608830+0000", 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0055657650000000003 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.620639+0000", 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011808712000000001 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.216 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.660037+0000", 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0393984 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.660195+0000", 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000157868 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.660243+0000", 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7407000000000002e-05 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.217 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45206.0:200325 7.d 7:b8fa8355:::benchmark_data_smithi032_200324_object75659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:17:48.622905+0000", 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 532.58211426599996, 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0374521329999999, 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45206", 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/2817492857", 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 200325 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.218 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.622905+0000", 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.622905+0000", 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.622905+0000", 2024-09-21T10:26:41.219 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.622910+0000", 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.84e-06 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.622920+0000", 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8959999999999998e-06 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.220 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.622925+0000", 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9620000000000001e-06 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.622963+0000", 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8219000000000001e-05 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:48.622978+0000", 2024-09-21T10:26:41.221 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4901e-05 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.605423+0000", 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98244492999999999 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.605548+0000", 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012498399999999999 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:26:41.222 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.605632+0000", 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3515000000000003e-05 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.612130+0000", 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0064984359999999998 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.629120+0000", 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.016989496 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.223 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.660288+0000", 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.031167765 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.660318+0000", 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0490000000000001e-05 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.224 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:17:49.660358+0000", 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9606e-05 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:30972 8.5 8:a0979ea6:::benchmark_data_smithi032_233595_object30971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:19:49.559472+0000", 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 411.64554819199998, 2024-09-21T10:26:41.225 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0727779919999998, 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 30972 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.559472+0000", 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.226 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.559472+0000", 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.559470+0000", 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.559530+0000", 2024-09-21T10:26:41.227 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9679999999999998e-05 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.559545+0000", 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5590000000000002e-05 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.559553+0000", 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8089999999999993e-06 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.559604+0000", 2024-09-21T10:26:41.228 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0992000000000002e-05 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.559619+0000", 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4908e-05 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.544158+0000", 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98453875499999999 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.544177+0000", 2024-09-21T10:26:41.229 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9332999999999999e-05 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.626181+0000", 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0820037469999999 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.626398+0000", 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000216894 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.230 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.626612+0000", 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00021466700000000001 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.630782+0000", 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0041700039999999997 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.630856+0000", 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3359e-05 2024-09-21T10:26:41.231 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.630988+0000", 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000131688 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.631037+0000", 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9150000000000002e-05 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.632250+0000", 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0012128799999999999 2024-09-21T10:26:41.232 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:30973 8.5 8:adb6be2a:::benchmark_data_smithi032_233595_object30972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:19:49.565588+0000", 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 411.63943154700002, 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0698018459999998, 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 30973 2024-09-21T10:26:41.233 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.565588+0000", 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.565588+0000", 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.565587+0000", 2024-09-21T10:26:41.234 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.565639+0000", 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2031999999999999e-05 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.565650+0000", 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1245e-05 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.565657+0000", 2024-09-21T10:26:41.235 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6510000000000004e-06 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.565698+0000", 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0640999999999999e-05 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.565705+0000", 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6729999999999992e-06 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.544208+0000", 2024-09-21T10:26:41.236 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97850318000000003 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.544219+0000", 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0907000000000001e-05 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.629172+0000", 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0849527860000001 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.629324+0000", 2024-09-21T10:26:41.237 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015164599999999999 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.629499+0000", 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017480100000000001 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.632274+0000", 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0027759460000000001 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.238 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.633982+0000", 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001707898 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.634053+0000", 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0751000000000001e-05 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.634120+0000", 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6759000000000002e-05 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.239 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.635390+0000", 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001270162 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:30974 8.5 8:a2ac69de:::benchmark_data_smithi032_233595_object30973:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:19:49.567190+0000", 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 411.637829764, 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.073250356, 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.240 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 30974 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.567190+0000", 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.567190+0000", 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.241 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.567189+0000", 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.567228+0000", 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.862e-05 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.567238+0000", 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8779999999999993e-06 2024-09-21T10:26:41.242 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.567243+0000", 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2170000000000002e-06 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.567276+0000", 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2929000000000003e-05 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:49.567283+0000", 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3599999999999998e-06 2024-09-21T10:26:41.243 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.544245+0000", 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.97696153900000005 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:50.544254+0000", 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0499999999999997e-06 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.632327+0000", 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0880729650000001 2024-09-21T10:26:41.244 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.632467+0000", 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013976200000000001 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.632581+0000", 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000114505 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.637022+0000", 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0044407140000000001 2024-09-21T10:26:41.245 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.637060+0000", 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8399999999999998e-05 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.638824+0000", 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001763571 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.246 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.638886+0000", 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2199999999999994e-05 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:19:51.640440+0000", 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015543810000000001 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:128257 8.9 8:9cc7b2fb:::benchmark_data_smithi032_233595_object28412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:00.058754+0000", 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 281.14626604099999, 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1556567649999998, 2024-09-21T10:26:41.247 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 128257 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.058754+0000", 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.058754+0000", 2024-09-21T10:26:41.248 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.058751+0000", 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999976 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.058762+0000", 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0002e-05 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.058778+0000", 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6898999999999999e-05 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.249 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.058787+0000", 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6090000000000001e-06 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.058860+0000", 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2791000000000002e-05 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.058874+0000", 2024-09-21T10:26:41.250 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3890000000000001e-05 2024-09-21T10:26:41.251 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:41.251 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.251 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:01.002223+0000", 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.94334935900000005 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:01.002237+0000", 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3817e-05 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.105370+0000", 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1031334699999999 2024-09-21T10:26:41.252 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for degraded object", 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.105791+0000", 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00042080600000000002 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.115198+0000", 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094071290000000002 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.115263+0000", 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4809999999999998e-05 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.253 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3,5", 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.115351+0000", 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8043999999999999e-05 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.118722+0000", 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0033706560000000001 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.118753+0000", 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0735000000000003e-05 2024-09-21T10:26:41.254 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.118812+0000", 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9503999999999998e-05 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.213203+0000", 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.094391427999999999 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.213271+0000", 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7152999999999999e-05 2024-09-21T10:26:41.255 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.214410+0000", 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0011398619999999999 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:128299 8.9 8:9c70cfd2:::benchmark_data_smithi032_233595_object28454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:00.359102+0000", 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 280.845917289, 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.856324992, 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.256 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 128299 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.359102+0000", 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.359102+0000", 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.257 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.359100+0000", 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999976 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.359111+0000", 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0353999999999999e-05 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.359131+0000", 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0448000000000001e-05 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.258 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.359140+0000", 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2830000000000007e-06 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.359206+0000", 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5117999999999996e-05 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.359219+0000", 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3801000000000001e-05 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.259 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:01.002267+0000", 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.643047437 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:01.002276+0000", 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.7280000000000007e-06 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.107124+0000", 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1048479840000001 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for degraded object", 2024-09-21T10:26:41.260 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.107606+0000", 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00048132600000000002 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.121903+0000", 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014297242999999999 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.122012+0000", 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010859200000000001 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3,5", 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.122155+0000", 2024-09-21T10:26:41.261 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014293300000000001 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.130563+0000", 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0084083179999999997 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.130575+0000", 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2018000000000001e-05 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.130629+0000", 2024-09-21T10:26:41.262 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4323000000000001e-05 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.214470+0000", 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.083840385000000003 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.214506+0000", 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6395e-05 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.263 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.215427+0000", 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000921458 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:128302 8.9 8:9958b095:::benchmark_data_smithi032_233595_object28457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:00.378754+0000", 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 280.82626598600001, 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.837769668, 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.264 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 128302 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.378754+0000", 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.378754+0000", 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.265 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.378752+0000", 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.378760+0000", 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5730000000000002e-06 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.378776+0000", 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5954999999999999e-05 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.266 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.378783+0000", 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9879999999999998e-06 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.378833+0000", 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0198000000000003e-05 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.378843+0000", 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0057999999999999e-05 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.267 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:01.002303+0000", 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.623460334 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:01.002312+0000", 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.02e-06 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.107751+0000", 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1054384930000001 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.268 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for degraded object", 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.108017+0000", 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00026630799999999999 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.127252+0000", 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.019235107000000001 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.127370+0000", 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000117995 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3,5", 2024-09-21T10:26:41.269 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.127552+0000", 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00018177800000000001 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.130702+0000", 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031502539999999999 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.136586+0000", 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0058833749999999997 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.270 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.180784+0000", 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.044198635 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.215497+0000", 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.034712337000000003 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.215545+0000", 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8625999999999998e-05 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.271 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.216523+0000", 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00097814100000000008 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:128311 8.9 8:94e20101:::benchmark_data_smithi032_233595_object28466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:00.485281+0000", 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 280.71973875999998, 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7324269800000001, 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.272 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 128311 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.485281+0000", 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.485281+0000", 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.485280+0000", 2024-09-21T10:26:41.273 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.485286+0000", 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.793e-06 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.485299+0000", 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2858e-05 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.485307+0000", 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9470000000000003e-06 2024-09-21T10:26:41.274 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.485356+0000", 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8681999999999999e-05 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:00.485366+0000", 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6350000000000001e-06 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:01.002341+0000", 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.51697532099999999 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.275 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:26:41.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:01.002350+0000", 2024-09-21T10:26:41.276 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.329e-06 2024-09-21T10:26:41.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.276 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.276 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.276 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.108071+0000", 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1057212160000001 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for degraded object", 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.108451+0000", 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000379537 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.132951+0000", 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.024500397 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.277 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.133064+0000", 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011286900000000001 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3,5", 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.133220+0000", 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015624200000000001 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.141915+0000", 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0086943160000000005 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.278 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.143787+0000", 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0018719749999999999 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.180866+0000", 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.037079203999999998 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.216586+0000", 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.035720487000000002 2024-09-21T10:26:41.279 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.216639+0000", 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2102999999999999e-05 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.217708+0000", 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0010694070000000001 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.280 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:198067 8.5 8:ac4a5855:::benchmark_data_smithi032_233595_object98222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:57.315087+0000", 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 163.889932595, 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.045247305000000002, 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 198067 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.281 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.315087+0000", 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.315087+0000", 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.315086+0000", 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.282 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.315094+0000", 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3350000000000007e-06 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.315112+0000", 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8281999999999999e-05 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.315121+0000", 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.653e-06 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.283 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.325620+0000", 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010499656 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.325729+0000", 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010870900000000001 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.326306+0000", 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00057717200000000004 2024-09-21T10:26:41.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.335920+0000", 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0096138460000000005 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.358865+0000", 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.022945011000000001 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.359212+0000", 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00034628000000000003 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.285 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.360154+0000", 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00094267199999999998 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.360334+0000", 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00018026699999999999 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:198070 8.5 8:a6e0957e:::benchmark_data_smithi032_233595_object98225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:57.316111+0000", 2024-09-21T10:26:41.286 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 163.88890854100001, 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.044809143000000003, 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 198070 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.316111+0000", 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.287 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.316111+0000", 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.316110+0000", 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.316127+0000", 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7025999999999999e-05 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.288 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.316141+0000", 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4174e-05 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.316149+0000", 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1740000000000003e-06 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.331518+0000", 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015369644999999999 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.289 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.331716+0000", 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00019809499999999999 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.331858+0000", 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014201599999999999 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.340425+0000", 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0085666270000000003 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.290 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.358893+0000", 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018467977999999999 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.360610+0000", 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0017169329999999999 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.360762+0000", 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015232599999999999 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.291 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.360920+0000", 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015816300000000001 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:198073 8.5 8:a8a85346:::benchmark_data_smithi032_233595_object98228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:57.317111+0000", 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 163.887908463, 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.064014418000000003, 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.292 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 198073 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.317111+0000", 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.317111+0000", 2024-09-21T10:26:41.293 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.317110+0000", 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.317116+0000", 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1769999999999997e-06 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.294 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.317129+0000", 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3267e-05 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.317136+0000", 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7440000000000001e-06 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.335943+0000", 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018807351999999999 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.295 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.336110+0000", 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000167417 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.336228+0000", 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000117971 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.358914+0000", 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.022686016999999999 2024-09-21T10:26:41.296 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.361104+0000", 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002189435 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381034+0000", 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.019930586 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381097+0000", 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2807999999999999e-05 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.297 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381126+0000", 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8561e-05 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:198079 8.5 8:a9b73454:::benchmark_data_smithi032_233595_object98234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:57.323306+0000", 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 163.88171324800001, 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.057959381999999997, 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.298 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 198079 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.323306+0000", 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.323306+0000", 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.299 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.323305+0000", 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.323312+0000", 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3829999999999997e-06 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.323323+0000", 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1089e-05 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.300 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.323330+0000", 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6299999999999998e-06 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.340538+0000", 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.017208116999999998 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.340853+0000", 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00031479199999999999 2024-09-21T10:26:41.301 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.341327+0000", 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00047430699999999998 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.360393+0000", 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.019065576000000001 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381005+0000", 2024-09-21T10:26:41.302 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.020611622 2024-09-21T10:26:41.303 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.303 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381169+0000", 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000164389 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381221+0000", 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1570999999999998e-05 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381266+0000", 2024-09-21T10:26:41.304 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5238000000000003e-05 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:198084 8.5 8:a70a0658:::benchmark_data_smithi032_233595_object98239:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:57.330648+0000", 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 163.87437135600001, 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.050811075999999997, 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.305 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 198084 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.330648+0000", 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.330648+0000", 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.306 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.330647+0000", 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.330654+0000", 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.7689999999999996e-06 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.330671+0000", 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6657999999999999e-05 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.330679+0000", 2024-09-21T10:26:41.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3410000000000003e-06 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.346497+0000", 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015817904000000001 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.346825+0000", 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00032774699999999998 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:26:41.308 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.349975+0000", 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031498279999999999 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.360422+0000", 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010447331000000001 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381152+0000", 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.020729656999999999 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.309 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381340+0000", 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000188271 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381422+0000", 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1515999999999997e-05 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.381459+0000", 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7756000000000002e-05 2024-09-21T10:26:41.310 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:198296 8.9 8:99979a2a:::benchmark_data_smithi032_233595_object98451:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:57.593496+0000", 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 163.61152413400001, 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.046165640000000001, 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 198296 2024-09-21T10:26:41.311 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.593496+0000", 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.593496+0000", 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.593494+0000", 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:41.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.593503+0000", 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5790000000000004e-06 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.593521+0000", 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8678999999999999e-05 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.593528+0000", 2024-09-21T10:26:41.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0620000000000003e-06 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.593583+0000", 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4907000000000001e-05 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.593710+0000", 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012711999999999999 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.593823+0000", 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000112859 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.600797+0000", 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0069739440000000001 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.600887+0000", 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9866999999999999e-05 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.639575+0000", 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.038688452999999998 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.639634+0000", 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8146000000000002e-05 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:57.639661+0000", 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7662999999999999e-05 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.316 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:95604 9.b 9:d1d4f128:::benchmark_data_smithi032_265250_object95603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:25:27.730276+0000", 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 73.474743485000005, 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.097772939000000003, 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 95604 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.730276+0000", 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.730276+0000", 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.730274+0000", 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:26:41.318 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.730304+0000", 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0303000000000001e-05 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.730322+0000", 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7399999999999999e-05 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.730331+0000", 2024-09-21T10:26:41.319 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7369999999999995e-06 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.730407+0000", 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5961e-05 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.730566+0000", 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000159541 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 5", 2024-09-21T10:26:41.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.730698+0000", 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013172499999999999 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.733900+0000", 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0032026070000000001 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.827856+0000", 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.093955082999999995 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.827945+0000", 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9235999999999997e-05 2024-09-21T10:26:41.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.828049+0000", 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000104431 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:95615 9.b 9:dfbfb0d3:::benchmark_data_smithi032_265250_object95614:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:25:27.773806+0000", 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 73.431213708000001, 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.054463546000000002, 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.322 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 95615 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.773806+0000", 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.773806+0000", 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.773804+0000", 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.773836+0000", 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1701e-05 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.773850+0000", 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4491000000000001e-05 2024-09-21T10:26:41.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.773857+0000", 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2809999999999997e-06 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.773902+0000", 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5290999999999997e-05 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.774013+0000", 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000111147 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 5", 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.774108+0000", 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5278000000000005e-05 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.776743+0000", 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0026344070000000001 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.828093+0000", 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.051349998000000001 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.828178+0000", 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5111999999999994e-05 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:25:27.828270+0000", 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.1593000000000002e-05 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.327 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:142512 9.b e394/390 9:d04f7ccf:::benchmark_data_smithi032_265250_object14710:head v 394'8817, mlcod=382'2647)", 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.103843+0000", 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 19.101176385999999, 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.28914228400000003, 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.103843+0000", 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.328 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.103843+0000", 2024-09-21T10:26:41.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.329 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.103842+0000", 2024-09-21T10:26:41.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:26:41.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.103853+0000", 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0575e-05 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.103867+0000", 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3927000000000001e-05 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.103874+0000", 2024-09-21T10:26:41.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4989999999999997e-06 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.103941+0000", 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6773999999999995e-05 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.103966+0000", 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4655e-05 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.392905+0000", 2024-09-21T10:26:41.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.28893907699999999 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.392986+0000", 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0881e-05 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:142517 9.1 e394/378 9:8cf625f9:::benchmark_data_smithi032_265250_object14715:head v 394'9009, mlcod=394'9007)", 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.106769+0000", 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 19.098250628999999, 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.280137786, 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.332 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.106769+0000", 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.106769+0000", 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.106768+0000", 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:41.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.106778+0000", 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8670000000000006e-06 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.381667+0000", 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.27488942599999999 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.381681+0000", 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4092e-05 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.381786+0000", 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010500799999999999 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.381810+0000", 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3983999999999999e-05 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.386881+0000", 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0050710030000000001 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.335 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.386907+0000", 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5639999999999998e-05 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:142515 9.b e394/390 9:d69212f7:::benchmark_data_smithi032_265250_object14713:head v 394'8818, mlcod=382'2647)", 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.110447+0000", 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 19.094572615000001, 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.29591233, 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:41.336 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.110447+0000", 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.110447+0000", 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.110446+0000", 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.110455+0000", 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3479999999999993e-06 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.381738+0000", 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.271282888 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.381747+0000", 2024-09-21T10:26:41.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9209999999999998e-06 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.384979+0000", 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0032323289999999999 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.384996+0000", 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6977e-05 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.406321+0000", 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.021324487 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.406359+0000", 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8699999999999999e-05 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:41.340 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:26:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:41.341 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:41.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:41.442 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:41.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:41.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:41.588 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:41.630 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:41.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:41.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:41.732 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:41.768 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:41.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:41.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:41.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:41.877 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:41 smithi032 ceph-mon[105748]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 16826/298779 objects degraded (5.632%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:26:41.877 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:41 smithi032 ceph-mon[101299]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 16826/298779 objects degraded (5.632%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:26:41.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:41.913 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:41.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:42.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:42.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:41 smithi045 ceph-mon[95976]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 16826/298779 objects degraded (5.632%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:26:42.060 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:42.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:42.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:42.162 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:42.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:42.198 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:42.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:42.299 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:42.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:26:42.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:42.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:42.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:42.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:42.496 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:42.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:42.597 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:42.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:42.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:42.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:42.698 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:42.727 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:26:42.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:42.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:42.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:42.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:42.879 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:42.924 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:42.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:43.026 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:42 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16826/298779 objects degraded (5.632%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:42 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/553268033' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:42 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:43.074 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:43.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:43.089 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:42 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16826/298779 objects degraded (5.632%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:43.089 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:42 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/553268033' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:43.089 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:42 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:43.089 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:42 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16826/298779 objects degraded (5.632%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:43.089 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:42 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/553268033' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:43.089 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:42 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:43.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:43.221 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:43.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:43.323 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:43.357 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:43.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:43.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:43.496 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:43.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:43.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:43.598 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:43.635 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:43.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:43.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:43.737 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:43.760 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:43.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:43.861 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:43.902 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:43.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:43.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:44.006 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:44.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:43 smithi045 ceph-mon[95976]: pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 16826/297384 objects degraded (5.658%); 2.0 MiB/s, 32 objects/s recovering 2024-09-21T10:26:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:43 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:26:44.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:43 smithi045 ceph-mon[95976]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T10:26:44.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:44.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:44.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:44.141 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:44.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:43 smithi032 ceph-mon[101299]: pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 16826/297384 objects degraded (5.658%); 2.0 MiB/s, 32 objects/s recovering 2024-09-21T10:26:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:43 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:26:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:43 smithi032 ceph-mon[101299]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T10:26:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:43 smithi032 ceph-mon[105748]: pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 16826/297384 objects degraded (5.658%); 2.0 MiB/s, 32 objects/s recovering 2024-09-21T10:26:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:43 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:26:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:43 smithi032 ceph-mon[105748]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T10:26:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:44.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:44.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:44.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:44.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:44.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:44.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:44.439 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:44.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:44.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:44.589 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:44.628 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:44.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:44.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:44.730 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:44.789 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:44.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:44.827 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:26:44.827 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:26:44.827 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.827 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:160387 9.9 e398/390 9:96e80880:::benchmark_data_smithi032_265250_object32585:head v 398'10095, mlcod=382'2729)", 2024-09-21T10:26:44.827 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.822574+0000", 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.001711356, 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0017413159999999999, 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.822574+0000", 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.822574+0000", 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.828 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.822573+0000", 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.822580+0000", 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7599999999999997e-06 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.822594+0000", 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3575e-05 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.829 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.822601+0000", 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9589999999999998e-06 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.822662+0000", 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1717999999999994e-05 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.822685+0000", 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2863000000000001e-05 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:160370 9.e e398/378)", 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.823214+0000", 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00107124, 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0011680320000000001, 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.831 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823214+0000", 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823214+0000", 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823213+0000", 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.832 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823219+0000", 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2780000000000001e-06 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823229+0000", 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0708e-05 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823236+0000", 2024-09-21T10:26:44.833 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3910000000000003e-06 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:160392 9.e 9.f4900c9e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.823482+0000", 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00080372400000000002, 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00094634700000000005, 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:44.834 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160392 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823482+0000", 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823482+0000", 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.835 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823480+0000", 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823487+0000", 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6610000000000003e-06 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823499+0000", 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2194999999999999e-05 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823505+0000", 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3659999999999997e-06 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:160382 9.d e398/378)", 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.823733+0000", 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00055228999999999999, 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00074575800000000001, 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.837 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823733+0000", 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823733+0000", 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823733+0000", 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.838 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823736+0000", 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7500000000000001e-06 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823744+0000", 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9889999999999992e-06 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.823749+0000", 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1619999999999999e-06 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.839 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:160365 9.e 9:7dbb19c4:::benchmark_data_smithi032_265250_object32563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.797578+0000", 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.026707336000000002, 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.026949220999999999, 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160365 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.840 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.841 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.797578+0000", 2024-09-21T10:26:44.841 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.841 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.841 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.841 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.841 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.797578+0000", 2024-09-21T10:26:44.841 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.797577+0000", 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.797584+0000", 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7780000000000002e-06 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.797598+0000", 2024-09-21T10:26:44.842 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4083000000000001e-05 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.797604+0000", 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.72e-06 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.797683+0000", 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8529999999999995e-05 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.797791+0000", 2024-09-21T10:26:44.843 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000108167 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.797895+0000", 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010385700000000001 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.811746+0000", 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013851525 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.811782+0000", 2024-09-21T10:26:44.844 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5565000000000003e-05 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:160367 9.e 9:71ecd1fb:::benchmark_data_smithi032_265250_object32565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.798530+0000", 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.025755389, 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.026077526, 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:44.845 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160367 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798530+0000", 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798530+0000", 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798529+0000", 2024-09-21T10:26:44.846 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798535+0000", 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0090000000000004e-06 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798552+0000", 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6846000000000001e-05 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798560+0000", 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6039999999999993e-06 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.847 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.804802+0000", 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0062421999999999998 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.804911+0000", 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0001087 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.805018+0000", 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000106808 2024-09-21T10:26:44.848 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815959+0000", 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010941377 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.819755+0000", 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0037952279999999999 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:160370 9.e 9:7e7e953e:::benchmark_data_smithi032_265250_object32568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.798843+0000", 2024-09-21T10:26:44.849 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.025442599999999999, 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.025843298000000001, 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160370 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798843+0000", 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.850 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798843+0000", 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798842+0000", 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798850+0000", 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4670000000000004e-06 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.851 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.852 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.852 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798866+0000", 2024-09-21T10:26:44.852 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5909999999999998e-05 2024-09-21T10:26:44.852 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.853 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.853 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.853 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.798874+0000", 2024-09-21T10:26:44.853 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.323e-06 2024-09-21T10:26:44.853 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.811803+0000", 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012929362 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.811926+0000", 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0001226 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:26:44.854 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.812048+0000", 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012235600000000001 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:160377 9.d 9:b4b0ed68:::benchmark_data_smithi032_265250_object32575:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.815473+0000", 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0088122819999999994, 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0092832969999999994, 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:44.855 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160377 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815473+0000", 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815473+0000", 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.856 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815471+0000", 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815479+0000", 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6029999999999999e-06 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815492+0000", 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3135999999999999e-05 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815500+0000", 2024-09-21T10:26:44.857 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3119999999999996e-06 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815878+0000", 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00037800699999999997 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815985+0000", 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000107178 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.858 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.816102+0000", 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000116889 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.824717+0000", 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0086158280000000007 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.824758+0000", 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1001000000000003e-05 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.859 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:160378 9.e 9:71b6d796:::benchmark_data_smithi032_265250_object32576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.815512+0000", 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.008773622, 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0093244139999999996, 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160378 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.860 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815512+0000", 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815512+0000", 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815511+0000", 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.861 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.862 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815514+0000", 2024-09-21T10:26:44.862 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5399999999999998e-06 2024-09-21T10:26:44.862 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.862 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.862 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.862 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815520+0000", 2024-09-21T10:26:44.862 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6110000000000001e-06 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815524+0000", 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3970000000000004e-06 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815979+0000", 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000455117 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.816078+0000", 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8577000000000002e-05 2024-09-21T10:26:44.863 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.816180+0000", 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000102456 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:160380 9.e 9:75f26f15:::benchmark_data_smithi032_265250_object32578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.815528+0000", 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0087574790000000003, 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0093761160000000003, 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.864 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160380 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815528+0000", 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815528+0000", 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.865 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815528+0000", 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815530+0000", 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0889999999999998e-06 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815535+0000", 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9599999999999999e-06 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.866 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815539+0000", 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0300000000000004e-06 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.819776+0000", 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0042376050000000002 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.819897+0000", 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000121111 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.867 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.820017+0000", 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0001197 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:160381 9.e 9.22f05e1e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.815542+0000", 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0087431469999999997, 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094268800000000003, 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:44.868 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160381 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815542+0000", 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815542+0000", 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815542+0000", 2024-09-21T10:26:44.869 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815545+0000", 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5469999999999999e-06 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815549+0000", 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.763e-06 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815553+0000", 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0260000000000001e-06 2024-09-21T10:26:44.870 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:160382 9.d 9:b01cdba2:::benchmark_data_smithi032_265250_object32580:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:44.815557+0000", 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0087288920000000002, 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094639830000000005, 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 160382 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:26:44.871 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815557+0000", 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815557+0000", 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815556+0000", 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.872 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815558+0000", 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.922e-06 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815563+0000", 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9089999999999999e-06 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.815567+0000", 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0679999999999998e-06 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.820395+0000", 2024-09-21T10:26:44.873 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0048278269999999998 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.820500+0000", 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000104849 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.820606+0000", 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000105943 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:44.824751+0000", 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0041451530000000004 2024-09-21T10:26:44.874 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.875 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:26:44.875 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.875 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:26:44.875 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:26:44.875 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 12 2024-09-21T10:26:44.875 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:26:44.891 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:44.928 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:44.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:44.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:45.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:45.061 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:45.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:45.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:45.163 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:45.200 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:45.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:45.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:45.302 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:45.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:45.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:45.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:45.465 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:45.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:45.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:45.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:45.640 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:45.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:45.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:45.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:45.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:45.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:45.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:45.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:45.918 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:45.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:45.923 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:26:45.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:45 smithi032 ceph-mon[101299]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 16826/292137 objects degraded (5.760%); 2.0 MiB/s, 32 objects/s recovering 2024-09-21T10:26:45.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:45 smithi032 ceph-mon[105748]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 16826/292137 objects degraded (5.760%); 2.0 MiB/s, 32 objects/s recovering 2024-09-21T10:26:46.019 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:46.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:45 smithi045 ceph-mon[95976]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 16826/292137 objects degraded (5.760%); 2.0 MiB/s, 32 objects/s recovering 2024-09-21T10:26:46.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:46.065 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:46.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:46.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:26:46.166 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:46.211 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:46.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:46.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:46.313 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:46.317 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:46.362 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:46.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:46.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:46.465 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:46.504 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:46.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:46.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:46.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:46.653 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:26:46.653 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:26:46.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:46.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:46.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:46.677 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:26:46.677 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:26:46.704 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:26:46.704 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:26:46.739 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:26:46.739 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:26:46.758 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:46.782 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:46.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:46.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:46.784 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:26:46.784 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:26:46.825 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:26:46.825 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:26:46.869 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:26:46.870 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:26:46.870 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:46 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1356930712' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:46.870 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:46 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1356930712' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:46.883 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:46.914 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:26:46.914 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:26:46.919 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:46.920 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:46.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:47.022 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:47.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:46 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1356930712' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:47.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:47.066 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:47.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:47.167 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:47.207 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:47.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:47.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:47.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:47.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:47.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:47.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:47.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:47.488 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:47.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:47.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:47.590 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:47.625 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:47.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:47.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:47.727 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:47.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:47.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:47.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:47.768 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:26:47.866 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:47.900 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:47.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:47.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:47.934 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:47 smithi045 ceph-mon[95976]: pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 16647/285249 objects degraded (5.836%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:26:47.934 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:47 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16647/285249 objects degraded (5.836%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:47.934 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:47 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/454480117' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:26:47.934 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:47 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:26:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:47 smithi032 ceph-mon[105748]: pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 16647/285249 objects degraded (5.836%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:26:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:47 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16647/285249 objects degraded (5.836%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:47 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/454480117' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:26:47.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:47 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:26:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:47 smithi032 ceph-mon[101299]: pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 16647/285249 objects degraded (5.836%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:26:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:47 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16647/285249 objects degraded (5.836%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:26:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:47 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/454480117' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:26:47.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:47 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:26:48.003 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:48.037 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:48.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:48.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:48.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:48.182 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:48.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:48.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:48.283 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:48.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:48.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:48.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:48.422 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:48.458 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:48.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:48.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:48.560 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:48.616 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:48.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:48.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:48.718 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:48.757 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:48.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:48.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:48.771 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:48 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:48.771 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:48 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:26:48.771 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:48 smithi045 ceph-mon[95976]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T10:26:48.771 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:48.852 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:48 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:48.852 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:48 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:26:48.852 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:48 smithi032 ceph-mon[101299]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T10:26:48.852 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:48.852 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:48 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:48.852 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:48 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:26:48.852 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:48 smithi032 ceph-mon[105748]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T10:26:48.853 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:48.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:48.895 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:48.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:48.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:48.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:49.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:49.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:49.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:49.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:49.166 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:49.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:49.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:49.268 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:49.311 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:49.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:49.361 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:26:49.361 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:26:49.412 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:49.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:49.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:49.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:49.548 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:49.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:49.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:49.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:49.692 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:49.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:26:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:26:49] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.51.0" 2024-09-21T10:26:49.730 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:49.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:49.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:49.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:49.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:49.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:49.989 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:50.029 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:50.029 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:49 smithi032 ceph-mon[105748]: pgmap v1227: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 16647/285249 objects degraded (5.836%); 1.4 MiB/s, 22 objects/s recovering 2024-09-21T10:26:50.029 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:49 smithi032 ceph-mon[101299]: pgmap v1227: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 16647/285249 objects degraded (5.836%); 1.4 MiB/s, 22 objects/s recovering 2024-09-21T10:26:50.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:50.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:50.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:49 smithi045 ceph-mon[95976]: pgmap v1227: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 16647/285249 objects degraded (5.836%); 1.4 MiB/s, 22 objects/s recovering 2024-09-21T10:26:50.091 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:26:50.131 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:50.131 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:26:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:50.179 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:50.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:50.280 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:50.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:50.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:50.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:50.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:50.439 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:50.474 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:50.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:50.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:50.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:50.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:50.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:50.649 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:50.733 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:50.784 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:50.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:50.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:50.886 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:50.927 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:50.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:51.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:51.074 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:51.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:51.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:51.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:51.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:51.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:51.339 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:26:51.349 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:51.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:51.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:51.451 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:51.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:51.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:51.599 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:51.649 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:51.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:51.750 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:51.773 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:26:51.791 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:51.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:51.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:51.893 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:51.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:51.932 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:51.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:52.033 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:52.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:51 smithi045 ceph-mon[95976]: pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 16479/274860 objects degraded (5.995%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:26:52.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:51 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1973062578' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:52.036 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:51 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:52.069 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:51 smithi032 ceph-mon[105748]: pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 16479/274860 objects degraded (5.995%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:26:52.069 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:51 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1973062578' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:52.069 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:51 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:52.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:51 smithi032 ceph-mon[101299]: pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 16479/274860 objects degraded (5.995%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:26:52.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:51 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1973062578' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:52.070 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:51 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:52.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:52.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:52.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:52.178 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:52.221 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:52.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:52.323 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:52.357 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:52.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:52.459 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:52.500 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:52.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:52.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:52.501 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:26:52.502 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:26:52.602 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:52.639 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:52.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:52.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:52.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:52.774 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:52.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:52.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:52.876 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:52.915 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:52.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:52.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:53.002 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:26:53.002 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:26:53.017 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:53.048 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:26:53.048 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:26:53.048 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:52 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:26:53.049 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:52 smithi032 ceph-mon[101299]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T10:26:53.049 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:52 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16479/274860 objects degraded (5.995%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:26:53.049 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:52 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:26:53.049 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:52 smithi032 ceph-mon[105748]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T10:26:53.049 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:52 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16479/274860 objects degraded (5.995%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:26:53.051 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:53.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:53.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:53.081 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:26:53.082 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:26:53.117 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:26:53.117 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:26:53.150 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:26:53.150 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:26:53.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:53.181 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:26:53.181 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:26:53.188 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:53.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:53.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:53.212 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:26:53.213 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:26:53.251 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:26:53.251 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:26:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:52 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:26:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:52 smithi045 ceph-mon[95976]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T10:26:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:52 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16479/274860 objects degraded (5.995%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:26:53.289 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:53.322 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:53.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:53.424 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:53.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:53.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:53.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:53.481 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:26:53.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:53.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:53.613 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:53.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:53.714 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:53.721 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:53.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:53.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:53.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:53.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:53.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:53.995 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:54.032 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:54.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:54.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:54.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:54.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:54.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:54.182 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:53 smithi032 ceph-mon[105748]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 16479/273417 objects degraded (6.027%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:26:54.182 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:53 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4134849370' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:54.185 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:53 smithi045 ceph-mon[95976]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 16479/273417 objects degraded (6.027%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:26:54.185 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:53 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4134849370' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:54.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:53 smithi032 ceph-mon[101299]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 16479/273417 objects degraded (6.027%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:26:54.185 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:53 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4134849370' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:26:54.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:54.313 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:54.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:54.415 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:54.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:54.459 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:54.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:54.478 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:54.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:54.596 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:54.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:54.697 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:54.729 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:54.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:54.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:54.831 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:54.871 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:54.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:54.973 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:55.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:55.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:55.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:55.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:55.163 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:55.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:55.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:55.265 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:55.307 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:55.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:55.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:55.409 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:55.432 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:55.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:55.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:55.534 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:55.577 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:55.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:55.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:55.679 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:55.714 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:55.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:55.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:55.852 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:55.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:55.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:55.954 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:55.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:55 smithi032 ceph-mon[105748]: pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 16479/267939 objects degraded (6.150%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:26:55.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:55 smithi032 ceph-mon[101299]: pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 16479/267939 objects degraded (6.150%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:26:55.994 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:55.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:55.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:56.096 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:56.144 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:56.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:56.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:56.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:26:56.247 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:55 smithi045 ceph-mon[95976]: pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 16479/267939 objects degraded (6.150%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:26:56.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:56.292 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:56.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:56.393 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:56.426 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:56.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:56.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:56.528 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:26:56.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:56.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:56.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:26:56.658 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:56.702 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:56.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:56.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:56.803 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:56.841 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:56.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:56.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:56.943 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:56.982 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:26:56.991 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:56.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:56.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:57.092 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:57.131 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:57.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:57.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:57.277 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:57.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:57.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:57.344 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:26:57.378 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:57.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:57.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:57.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:57.540 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:57.569 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:57.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:57.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:57.671 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:57.718 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:57.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:57.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:57.820 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:57.858 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:57.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:57 smithi032 ceph-mon[101299]: pgmap v1232: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 16307/261360 objects degraded (6.239%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:26:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:57 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16307/261360 objects degraded (6.239%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:26:57.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:57 smithi032 ceph-mon[105748]: pgmap v1232: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 16307/261360 objects degraded (6.239%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:26:57.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:57 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16307/261360 objects degraded (6.239%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:26:57.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:57.997 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:57.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:57.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:57 smithi045 ceph-mon[95976]: pgmap v1232: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 16307/261360 objects degraded (6.239%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:26:58.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:57 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16307/261360 objects degraded (6.239%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:26:58.089 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:58.099 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:58.134 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:58.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:58.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:58.236 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:58.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:58.279 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:58.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:58.380 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:26:58.408 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:58.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:58.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:26:58.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:26:58.559 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:58.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:58.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:26:58.661 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:58.683 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:58.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:58.785 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:58.825 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:58.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:58.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:58.827 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:26:58.829 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:26:58.926 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:58.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:58 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2525793378' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:26:58.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:58 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1095494573' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:58.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:58 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:58 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2525793378' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:26:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:58 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1095494573' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:58.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:58 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:58.959 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:58.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:58.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:59.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:58 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2525793378' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:26:59.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:58 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1095494573' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:59.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:58 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:59.061 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:59.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:59.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:59.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:59.209 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:59.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:59.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:59.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:59.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:26:59.378 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:59.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:59.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:26:59.480 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:26:59.508 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:59.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:26:59.610 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:26:59.667 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:59.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:59.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:26:59.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:26:59] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.51.0" 2024-09-21T10:26:59.769 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:26:59.809 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:26:59.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:59.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:26:59.911 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:26:59.912 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 7 1 2024-09-21T10:26:59.947 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:26:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:26:59.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:26:59.979 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:59 smithi045 ceph-mon[95976]: pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 16307/261360 objects degraded (6.239%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:26:59.979 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:59 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:26:59.979 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2525793378' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:26:59.979 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:59 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:26:59.979 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:26:59 smithi045 ceph-mon[95976]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T10:27:00.049 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:00.091 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:00.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:00.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:00.193 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[101299]: pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 16307/261360 objects degraded (6.239%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:27:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2525793378' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:27:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:27:00.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[101299]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T10:27:00.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[105748]: pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 16307/261360 objects degraded (6.239%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:27:00.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:00.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2525793378' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:27:00.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:27:00.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:26:59 smithi032 ceph-mon[105748]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T10:27:00.226 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:00.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:00.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:00.305 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:27:00.305 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:27:00.305 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:172732 9.9 9:9b68c36e:::benchmark_data_smithi032_265250_object44930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.278831+0000", 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.023004030000000002, 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.023034419, 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 172732 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.306 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.278831+0000", 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.278831+0000", 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.278831+0000", 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:27:00.307 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.278833+0000", 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4700000000000001e-06 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.278839+0000", 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.614e-06 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.278843+0000", 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4689999999999999e-06 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.308 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.284067+0000", 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0052237899999999999 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.284202+0000", 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013452000000000001 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.284340+0000", 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013804099999999999 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.309 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.297677+0000", 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013337623999999999 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.297686+0000", 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6349999999999995e-06 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301365+0000", 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003679295 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.310 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301523+0000", 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015781200000000001 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:172734 9.5 9:a54ed6f6:::benchmark_data_smithi032_265250_object44932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.283431+0000", 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.018404229000000001, 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.018533838, 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.311 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 172734 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.283431+0000", 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.312 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.283431+0000", 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.283429+0000", 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.283436+0000", 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.216e-06 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.313 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.283450+0000", 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3705e-05 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.283459+0000", 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4789999999999996e-06 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.285176+0000", 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0017174029999999999 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.285321+0000", 2024-09-21T10:27:00.314 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014479 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.285480+0000", 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000158839 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.296028+0000", 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010548551999999999 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.315 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:172737 9.5 9:a310fd3f:::benchmark_data_smithi032_265250_object44935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.286190+0000", 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.015644675, 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015848649999999999, 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 172737 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286190+0000", 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.316 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286190+0000", 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286189+0000", 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286196+0000", 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2860000000000002e-06 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.318 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.318 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286207+0000", 2024-09-21T10:27:00.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.081e-05 2024-09-21T10:27:00.318 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.318 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.318 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.318 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286214+0000", 2024-09-21T10:27:00.318 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4660000000000002e-06 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300102+0000", 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.01388754 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300811+0000", 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00070921500000000002 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300974+0000", 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000162998 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:172718 9.d e401/378 9:be654a63:::benchmark_data_smithi032_265250_object44916:head v 401'10869, mlcod=401'10864)", 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.286245+0000", 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.015589881999999999, 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015864156000000001, 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.320 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286245+0000", 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286245+0000", 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286245+0000", 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286251+0000", 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1009999999999997e-06 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286258+0000", 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1219999999999997e-06 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.322 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.286263+0000", 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5990000000000001e-06 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.290325+0000", 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0040612230000000001 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.290344+0000", 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9619999999999998e-05 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.323 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:172709 9.5 e401/390 ondisk, result = 0)", 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.289784+0000", 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.012050709, 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012378491, 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289784+0000", 2024-09-21T10:27:00.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289784+0000", 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289784+0000", 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289787+0000", 2024-09-21T10:27:00.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4779999999999998e-06 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289793+0000", 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1630000000000001e-06 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289797+0000", 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1030000000000004e-06 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.293747+0000", 2024-09-21T10:27:00.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039501909999999996 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.293758+0000", 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.094e-05 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:172739 9.7 9:e00b5064:::benchmark_data_smithi032_265250_object44937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.289846+0000", 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.011989345, 2024-09-21T10:27:00.327 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012373467000000001, 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 172739 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289846+0000", 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.328 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289846+0000", 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289845+0000", 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289849+0000", 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4959999999999998e-06 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.329 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289858+0000", 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.081e-06 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289864+0000", 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7819999999999999e-06 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.289908+0000", 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3701000000000001e-05 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.290028+0000", 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000119982 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.290147+0000", 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000119382 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:172742 9.7 9:efc88bbf:::benchmark_data_smithi032_265250_object44940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.291100+0000", 2024-09-21T10:27:00.331 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.010735534999999999, 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011188138, 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 172742 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.291100+0000", 2024-09-21T10:27:00.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.291100+0000", 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.291098+0000", 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.291104+0000", 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2229999999999998e-06 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.291114+0000", 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0621e-05 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.291122+0000", 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4830000000000002e-06 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.299187+0000", 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0080652950000000001 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.299430+0000", 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024285100000000001 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.299614+0000", 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00018402499999999999 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.335 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:172727 9.7 e401/378)", 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.294957+0000", 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00687763, 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0073953730000000002, 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.294957+0000", 2024-09-21T10:27:00.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.294957+0000", 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.294957+0000", 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.294961+0000", 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4340000000000002e-06 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.294971+0000", 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8449999999999998e-06 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.294977+0000", 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6080000000000003e-06 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.338 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:172739 9.7 e401/378)", 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.295236+0000", 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0065988139999999997, 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0071667950000000001, 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295236+0000", 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295236+0000", 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295236+0000", 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295239+0000", 2024-09-21T10:27:00.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.388e-06 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295255+0000", 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5551000000000001e-05 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295261+0000", 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7499999999999997e-06 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.341 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:172739 9.7 e401/378)", 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.295355+0000", 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0064800760000000004, 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0070893029999999999, 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295355+0000", 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.342 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295355+0000", 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295354+0000", 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295359+0000", 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1749999999999998e-06 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295367+0000", 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4579999999999993e-06 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295386+0000", 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9071000000000001e-05 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:172736 9.d e401/378 9:bb242695:::benchmark_data_smithi032_265250_object44934:head v 401'10870, mlcod=401'10866)", 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.295403+0000", 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.006431984, 2024-09-21T10:27:00.344 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0070862520000000003, 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295403+0000", 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295403+0000", 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295403+0000", 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295408+0000", 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.028e-06 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.346 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:00.347 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295414+0000", 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2759999999999997e-06 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.295418+0000", 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4259999999999996e-06 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.296047+0000", 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00062882599999999997 2024-09-21T10:27:00.348 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.296062+0000", 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4973999999999999e-05 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:172727 9.7 9:e52a7a6a:::benchmark_data_smithi032_265250_object44925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.274216+0000", 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.027618983999999999, 2024-09-21T10:27:00.349 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.028328952000000001, 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 172727 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.274216+0000", 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.350 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.274216+0000", 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.274215+0000", 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.351 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.274219+0000", 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7780000000000001e-06 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.274228+0000", 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5599999999999994e-06 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.274233+0000", 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0939999999999998e-06 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.352 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.283351+0000", 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0091181830000000002 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.283605+0000", 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00025440499999999997 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.283748+0000", 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000143041 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.353 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.288778+0000", 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0050291850000000003 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.288909+0000", 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013106899999999999 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.354 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:172746 9.7 9.cdd5b3e7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.300171+0000", 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.001664319, 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024503760000000002, 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 172746 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300171+0000", 2024-09-21T10:27:00.355 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300171+0000", 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300169+0000", 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300176+0000", 2024-09-21T10:27:00.356 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0220000000000001e-06 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300186+0000", 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0481e-05 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300193+0000", 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0060000000000003e-06 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.357 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:172747 9.a 9:53de9e7d:::benchmark_data_smithi032_265250_object44945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.300208+0000", 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.001626972, 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024644710000000002, 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 172747 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.358 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300208+0000", 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300208+0000", 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300208+0000", 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.359 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300210+0000", 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.926e-06 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300221+0000", 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0233e-05 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300226+0000", 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2290000000000002e-06 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.360 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300273+0000", 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6931000000000002e-05 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300383+0000", 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000110394 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.300492+0000", 2024-09-21T10:27:00.361 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010861099999999999 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:172748 9.9 9.ae65a349 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.301145+0000", 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00069014200000000003, 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001624068, 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:27:00.362 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 172748 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301145+0000", 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301145+0000", 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.363 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301144+0000", 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301149+0000", 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1590000000000001e-06 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.364 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301162+0000", 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2964e-05 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301169+0000", 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9450000000000004e-06 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:172732 9.9 e401/390 ondisk, result = 0)", 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.301196+0000", 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00063895200000000003, 2024-09-21T10:27:00.365 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001622018, 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301196+0000", 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301196+0000", 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.366 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301195+0000", 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301199+0000", 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7139999999999999e-06 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301206+0000", 2024-09-21T10:27:00.367 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9210000000000003e-06 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301211+0000", 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8180000000000003e-06 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301350+0000", 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000139028 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:27:00.368 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301359+0000", 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.772e-06 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:172738 9.d e401/378)", 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:00.301216+0000", 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00061921600000000002, 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0016559280000000001, 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301216+0000", 2024-09-21T10:27:00.369 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301216+0000", 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301216+0000", 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967296 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301219+0000", 2024-09-21T10:27:00.370 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7869999999999999e-06 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301225+0000", 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3079999999999998e-06 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:00.301228+0000", 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7249999999999999e-06 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:27:00.371 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:27:00.372 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 17 2024-09-21T10:27:00.372 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:27:00.383 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:00.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:00.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:00.485 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:00.519 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:00.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:00.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:00.621 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:00.660 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:00.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:00.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:00.762 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:00.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:00.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:00.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:00.897 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:00.935 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:00.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:01.037 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:01.070 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:01.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:01.172 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:01.214 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:01.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:01.238 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:27:01.314 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:01.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:01.391 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:01.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:01.492 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:01.536 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:01.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:01.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:01.612 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:27:01.624 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:01.638 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:01.668 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:01.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:01.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:01.677 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:27:01.678 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:27:01.727 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:27:01.727 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:27:01.760 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:27:01.760 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:27:01.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:01.805 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:27:01.805 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:27:01.813 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:01.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:01.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:01.836 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:27:01.836 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:27:01.871 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:27:01.871 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:27:01.902 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:27:01.902 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:27:01.915 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:01.944 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:27:01.944 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:27:01.980 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:01.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:02.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:01 smithi045 ceph-mon[95976]: pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 16307/255912 objects degraded (6.372%); 1.2 MiB/s, 19 objects/s recovering 2024-09-21T10:27:02.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:01 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/48165872' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:02.081 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:02.120 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:02.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:02.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:02.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:01 smithi032 ceph-mon[101299]: pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 16307/255912 objects degraded (6.372%); 1.2 MiB/s, 19 objects/s recovering 2024-09-21T10:27:02.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:01 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/48165872' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:02.148 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:01 smithi032 ceph-mon[105748]: pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 16307/255912 objects degraded (6.372%); 1.2 MiB/s, 19 objects/s recovering 2024-09-21T10:27:02.148 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:01 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/48165872' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:02.221 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:02.252 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:02.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:02.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:02.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:02.391 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:02.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:02.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:02.492 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:02.528 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:02.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:02.534 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:27:02.629 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:02.702 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:02.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:02.803 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:02.842 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:02.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:02.943 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:02.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:02 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16307/255912 objects degraded (6.372%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:02.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:02 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16307/255912 objects degraded (6.372%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:02.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:02.999 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:02.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:03.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:02 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16307/255912 objects degraded (6.372%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:03.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:03.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:03.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:03.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:03.250 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:03.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:03.290 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:03.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:03.391 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:03.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:03.432 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:03.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:03.533 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:03.566 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:27:03.566 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:27:03.579 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:03.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:03.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:03.681 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:03.723 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:03.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:03.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:03.825 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:03.861 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:03.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:03.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:03 smithi032 ceph-mon[105748]: pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 16124/249381 objects degraded (6.466%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:27:03.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:03.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:03 smithi032 ceph-mon[101299]: pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 16124/249381 objects degraded (6.466%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:27:03.958 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:03.963 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:04.000 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:04.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:04.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:04.102 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:04.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:04.164 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:04.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:04.240 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:04.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:04.272 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:03 smithi045 ceph-mon[95976]: pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 16124/249381 objects degraded (6.466%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:27:04.272 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:04.304 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:27:04.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:04.431 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:04.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:04.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:04.574 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:04.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:04.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:04.666 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:27:04.676 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:04.729 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:04.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:04.831 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:04.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:04.867 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:04.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:04.880 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.7 to 1 (10000) 2024-09-21T10:27:04.887 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:04 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/642024242' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:27:04.887 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:04 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:27:04.967 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:05.005 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:05.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:05.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:05.107 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:05.135 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:04 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/642024242' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:27:05.135 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:04 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:27:05.140 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:05.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:05.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:04 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/642024242' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:27:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:04 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:27:05.241 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:05.274 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:05.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:05.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:05.375 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:05.417 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:05.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:05.519 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:05.564 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:05.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:05.603 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:27:05.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:05.701 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:05.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:05.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:05.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:05.833 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:05.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:05.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:05.935 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:05 smithi032 ceph-mon[101299]: pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 16124/244317 objects degraded (6.600%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:27:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:05 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:27:05.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:05 smithi032 ceph-mon[101299]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T10:27:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:05 smithi032 ceph-mon[105748]: pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 16124/244317 objects degraded (6.600%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:27:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:05 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:27:05.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:05 smithi032 ceph-mon[105748]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T10:27:05.984 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:05.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:05.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:06.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:06.117 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:06.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:06.173 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:05 smithi045 ceph-mon[95976]: pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 16124/244317 objects degraded (6.600%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:27:06.173 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:05 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:27:06.173 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:05 smithi045 ceph-mon[95976]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T10:27:06.219 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:06.263 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:06.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:06.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:06.364 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:06.414 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:06.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:06.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:27:06.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:06.570 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:06.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:06.671 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:06.680 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:27:06.681 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:27:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:06.734 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:06.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:06.835 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:06.867 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:06.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:06.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:06.969 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:07.012 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:07.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:07.066 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:07.113 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:07.151 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:07.180 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:07.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:07.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:07.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:06 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16124/244317 objects degraded (6.600%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:07.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:06 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16124/244317 objects degraded (6.600%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:07.282 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:06 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16124/244317 objects degraded (6.600%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:07.332 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:07.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:07.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:27:07.433 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:07.468 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:07.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:07.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:07.569 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:07.631 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:07.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:07.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:07.733 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:07.768 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:07.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:07.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:07.870 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:07.912 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:07.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:07.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:08.014 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:08.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:08.048 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:08.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:08.149 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:08.169 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:07 smithi032 ceph-mon[105748]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 989 op/s; 15930/237606 objects degraded (6.704%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T10:27:08.169 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:07 smithi032 ceph-mon[101299]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 989 op/s; 15930/237606 objects degraded (6.704%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T10:27:08.184 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:08.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:08.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:07 smithi045 ceph-mon[95976]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 989 op/s; 15930/237606 objects degraded (6.704%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T10:27:08.285 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:08.331 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:08.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:08.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:08.432 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:08.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:08.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:08.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:08.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:08.604 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:08.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:08.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:08.706 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:08.751 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:08.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:08.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:08.853 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:08.884 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:08.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:08.914 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:27:08.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:09.021 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:09.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:09.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:09.168 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:09.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:09.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:09.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:08 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/122625938' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:27:09.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:08 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/122625938' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:27:09.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:09.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:08 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/122625938' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:27:09.309 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:09.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:09.411 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:09.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:09.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:09.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:09.557 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:09.601 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:09.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:09.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:09.703 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:09.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:27:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:27:09] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-21T10:27:09.747 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:09.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:09.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:09.849 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:09.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:09.886 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:09.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:09.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:10.026 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:10.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:10.127 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:10.181 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:10.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:10.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:09 smithi032 ceph-mon[101299]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 15930/237606 objects degraded (6.704%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:27:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:09 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:27:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/122625938' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:27:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:09 smithi032 ceph-mon[101299]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T10:27:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:09 smithi032 ceph-mon[105748]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 15930/237606 objects degraded (6.704%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:27:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:09 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:27:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/122625938' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:27:10.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:09 smithi032 ceph-mon[105748]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T10:27:10.283 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:09 smithi045 ceph-mon[95976]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 15930/237606 objects degraded (6.704%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:27:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:09 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:27:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/122625938' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:27:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:09 smithi045 ceph-mon[95976]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T10:27:10.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:10.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:10.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:10.472 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:10.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:10.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:10.574 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:10.623 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:10.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:10.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:10.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:10.766 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:10.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:10.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:10.868 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:10.914 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:10.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:11.015 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:11.052 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:11.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:11.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:11.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:11.198 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:11.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:11.203 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:27:11.204 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:11.204 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:11.204 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:27:11.204 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:11.204 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:27:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:11.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:11.285 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:27:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=infra.usagestats t=2024-09-21T10:27:11.143600354Z level=info msg="Usage stats are ready to report" 2024-09-21T10:27:11.300 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:11.341 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:11.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:11.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:11.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:11.481 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:11.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:11.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:11.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:11.625 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:11.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:11.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:11.727 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:11.768 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:11.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:11.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:11.870 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:11.924 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:11.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:12.024 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:12.063 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:12.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:12.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:12.202 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:12.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:11 smithi032 ceph-mon[101299]: pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 15930/232326 objects degraded (6.857%); 1.5 MiB/s, 24 objects/s recovering 2024-09-21T10:27:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:11 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 15930/232326 objects degraded (6.857%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:11 smithi032 ceph-mon[105748]: pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 15930/232326 objects degraded (6.857%); 1.5 MiB/s, 24 objects/s recovering 2024-09-21T10:27:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:11 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 15930/232326 objects degraded (6.857%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:12.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:11 smithi045 ceph-mon[95976]: pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 15930/232326 objects degraded (6.857%); 1.5 MiB/s, 24 objects/s recovering 2024-09-21T10:27:12.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:11 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 15930/232326 objects degraded (6.857%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:12.305 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:12.353 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:12.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:12.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:12.455 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:12.506 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:12.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:12.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:12.607 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:12.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:12.651 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:12.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:12.752 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:12.779 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:12.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:12.881 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:12.914 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:12.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:12.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:13.016 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:13.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:13.058 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:13.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:13.159 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:13.183 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:13.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:13.284 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:13.320 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:13.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:13.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:13.422 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:13.456 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:13.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:13.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:13.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:13.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:13.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:13.693 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:13.730 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:13.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:13.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:13.854 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:13.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:13.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:13.956 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:13.996 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:13.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:13.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:14.097 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:14.157 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:14.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:14.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:13 smithi032 ceph-mon[105748]: pgmap v1243: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 15719/225297 objects degraded (6.977%); 3.2 MiB/s, 50 objects/s recovering 2024-09-21T10:27:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:13 smithi032 ceph-mon[101299]: pgmap v1243: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 15719/225297 objects degraded (6.977%); 3.2 MiB/s, 50 objects/s recovering 2024-09-21T10:27:14.259 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:13 smithi045 ceph-mon[95976]: pgmap v1243: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 15719/225297 objects degraded (6.977%); 3.2 MiB/s, 50 objects/s recovering 2024-09-21T10:27:14.292 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:14.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:14.394 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:14.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:14.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:14.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:14.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:14.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:14.600 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:14.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:14.701 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:14.728 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:27:14.728 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:27:14.737 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:14.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:14.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:14.762 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:27:14.762 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:27:14.793 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:27:14.793 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:27:14.823 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:27:14.823 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:27:14.839 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:14.866 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:27:14.866 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:27:14.878 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:14.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:14.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:14.900 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:27:14.900 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:27:14.938 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:27:14.938 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:27:14.980 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:15.001 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:27:15.001 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:27:15.001 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:14 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3412531790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:15.002 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:14 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3412531790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:15.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:15.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:15.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:15.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:15.166 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:15.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:15.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:15.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:15.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:14 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3412531790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:15.291 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:15.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:15.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:15.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:15.415 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:15.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:15.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:15.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:15.554 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:15.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:15.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:15.656 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:15.690 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:15.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:15.792 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:15.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:15.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:15.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:15.946 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:15 smithi032 ceph-mon[101299]: pgmap v1244: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 15719/219936 objects degraded (7.147%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:27:15.989 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:15.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:15.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:16.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:16.136 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:16.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:16.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:15 smithi032 ceph-mon[105748]: pgmap v1244: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 15719/219936 objects degraded (7.147%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:27:16.238 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:16.277 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:16.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:16.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:15 smithi045 ceph-mon[95976]: pgmap v1244: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 15719/219936 objects degraded (7.147%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:27:16.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:16.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:16.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:16.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:16.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:16.586 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:16.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:16.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:16.688 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:16.723 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:16.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:16.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:16.824 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:16.857 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:16.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:16.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:16.959 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:16.960 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:27:16.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:16.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:17.089 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:17.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:17.124 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:17.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:17.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:16 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3392817197' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:27:17.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:16 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:27:17.220 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:16 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 15719/219936 objects degraded (7.147%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:17.220 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:16 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3392817197' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:27:17.220 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:16 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:27:17.220 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:16 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 15719/219936 objects degraded (7.147%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:17.225 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:17.256 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:17.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:17.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:17.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:16 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3392817197' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:27:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:16 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:27:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:16 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 15719/219936 objects degraded (7.147%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:17.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:17.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:17.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:17.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:17.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:17.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:17.551 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:17.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:17.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:17.684 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:17.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:17.694 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:17.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 0 1 2024-09-21T10:27:17.786 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:17.818 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:17.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:17.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:17.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:17.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:17.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:18.051 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:18.073 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:18.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:18.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:18.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:18.199 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:18.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:18.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:18.218 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[105748]: pgmap v1245: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 15474/213558 objects degraded (7.246%); 2.8 MiB/s, 45 objects/s recovering 2024-09-21T10:27:18.218 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:27:18.218 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[105748]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T10:27:18.218 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:18.218 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:18.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[101299]: pgmap v1245: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 15474/213558 objects degraded (7.246%); 2.8 MiB/s, 45 objects/s recovering 2024-09-21T10:27:18.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:27:18.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[101299]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T10:27:18.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:18.219 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:17 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:17 smithi045 ceph-mon[95976]: pgmap v1245: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 15474/213558 objects degraded (7.246%); 2.8 MiB/s, 45 objects/s recovering 2024-09-21T10:27:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:17 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:27:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:17 smithi045 ceph-mon[95976]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T10:27:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:17 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:17 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:18.300 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:18.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:18.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:18.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:18.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:18.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:18.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:18.561 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:27:18.603 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:18.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:18.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:18.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:18.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:18.794 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:18.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:18.896 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:18.942 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:18.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:19.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:27:19.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:19.122 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:19.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:19.142 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:19.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:19.243 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:19.274 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:19.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:19.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:19.376 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:19.403 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:19.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:19.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:19.507 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:19.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:19.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:19.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:19.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:19.684 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:19.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:19.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:19.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:19.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:19 smithi032 ceph-mon[105748]: pgmap v1247: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 15474/213558 objects degraded (7.246%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:27:19.708 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:27:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:27:19] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-09-21T10:27:19.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:19.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:19.708 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:19 smithi032 ceph-mon[101299]: pgmap v1247: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 15474/213558 objects degraded (7.246%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:27:19.786 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:19.822 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:19.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:19.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:19.924 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:19.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:19.995 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:19.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:19 smithi045 ceph-mon[95976]: pgmap v1247: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 15474/213558 objects degraded (7.246%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:27:20.096 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:20.137 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:20.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:20.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:20.239 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:20.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:20.275 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:20.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:20.376 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:20.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:20.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:20.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:20.516 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:20.545 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:20.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:20.647 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:20.682 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:20.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:20.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:20.784 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:20.821 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:20.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:20.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:20.923 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:20.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:20.961 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:20.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:21.061 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:21.099 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:21.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:21.187 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:21.187 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:21.187 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:21.187 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:21.201 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:21.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:21.247 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:21.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:21.335 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:21.348 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:21.365 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:27:21.365 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:27:21.365 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:21.365 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:188928 9.9 e404/390 9:94494e9d:::benchmark_data_smithi032_265250_object61126:head v 404'11901, mlcod=382'2729)", 2024-09-21T10:27:21.365 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:21.354802+0000", 2024-09-21T10:27:21.365 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0075488719999999999, 2024-09-21T10:27:21.365 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0075800609999999999, 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:21.354802+0000", 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:21.354802+0000", 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:21.366 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:21.354803+0000", 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4470000000000001e-06 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:21.354813+0000", 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.9709999999999999e-06 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:21.354835+0000", 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1665999999999999e-05 2024-09-21T10:27:21.367 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:21.354845+0000", 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0022e-05 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:21.354908+0000", 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3749000000000002e-05 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:21.354936+0000", 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.7688999999999999e-05 2024-09-21T10:27:21.368 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:21.369 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:27:21.369 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:21.369 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:21.369 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:27:21.369 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 1 2024-09-21T10:27:21.369 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:27:21.397 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:21.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:21.499 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:21.542 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:21.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:21.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:21.643 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:21.681 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:21.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:21.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:21.782 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:21.831 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:21.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:21.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:22.007 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:22.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:22.107 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:22.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:22.153 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:22.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:22.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:22.296 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:22.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:22.332 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:22 smithi045 ceph-mon[95976]: pgmap v1248: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 15474/209052 objects degraded (7.402%); 2.8 MiB/s, 45 objects/s recovering 2024-09-21T10:27:22.332 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 15474/209052 objects degraded (7.402%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:22.398 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:22.448 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:22.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:22.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:22 smithi032 ceph-mon[105748]: pgmap v1248: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 15474/209052 objects degraded (7.402%); 2.8 MiB/s, 45 objects/s recovering 2024-09-21T10:27:22.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 15474/209052 objects degraded (7.402%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:22 smithi032 ceph-mon[101299]: pgmap v1248: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 15474/209052 objects degraded (7.402%); 2.8 MiB/s, 45 objects/s recovering 2024-09-21T10:27:22.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 15474/209052 objects degraded (7.402%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:22.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:22.583 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:22.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:22.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:22.685 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:22.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:22.723 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:22.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:22.824 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:22.862 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:22.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:22.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:22.964 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:22.982 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:27:22.993 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:22.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:22.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:23.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:23.096 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.0 to 1 (10000) 2024-09-21T10:27:23.121 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:23.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:23.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1812929649' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:27:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1812929649' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:27:23.222 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:23.246 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:23.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:23.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:23.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:23.388 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:23.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:23.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:23.489 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:23.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:23.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:23.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1812929649' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:27:23.626 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:23.667 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:23.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:23.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:23.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:23.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:23.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:23.942 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:23.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:23.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:24.044 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:24.075 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:24.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:24.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:24.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:24.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:24.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:24.320 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:24.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:27:24.342 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:24 smithi032 ceph-mon[105748]: pgmap v1249: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 15257/202905 objects degraded (7.519%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:27:24.343 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:24 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1812929649' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:27:24.343 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:24 smithi032 ceph-mon[105748]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T10:27:24.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:24 smithi032 ceph-mon[101299]: pgmap v1249: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 15257/202905 objects degraded (7.519%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:27:24.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:24 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1812929649' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:27:24.343 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:24 smithi032 ceph-mon[101299]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T10:27:24.368 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:24.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:24.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:24.398 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:24.408 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:24.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:24.499 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:24.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:24.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:24.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:24 smithi045 ceph-mon[95976]: pgmap v1249: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 15257/202905 objects degraded (7.519%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:27:24.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:24 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1812929649' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:27:24.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:24 smithi045 ceph-mon[95976]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T10:27:24.601 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:24.631 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:24.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:24.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:24.733 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:24.773 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:24.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:24.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:24.874 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:24.911 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:24.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:24.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:25.013 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:25.045 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:25.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:25.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:25.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:25.200 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:25.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:25.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:25.302 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:25.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:25.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:25.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:25.438 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:25.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:25.484 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:25.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:25.585 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:25.618 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:25.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:25.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:25.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:25.764 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:25.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:25.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:25.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:25.907 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:25.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:25.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:26.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:26.047 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:26.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:26.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:26.149 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:26.186 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:26.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:26.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:26.288 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:26.346 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:26.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:26.361 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:26 smithi032 ceph-mon[105748]: pgmap v1251: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 15257/197823 objects degraded (7.712%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T10:27:26.362 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:26 smithi032 ceph-mon[101299]: pgmap v1251: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 15257/197823 objects degraded (7.712%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T10:27:26.449 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:26.491 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:26.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:26.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:26 smithi045 ceph-mon[95976]: pgmap v1251: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 15257/197823 objects degraded (7.712%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T10:27:26.593 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:26.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:26.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:26.723 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:26.763 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:26.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:26.841 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:26.865 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:26.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:26.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:26.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:26.997 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:27.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:27.039 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:27.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:27.111 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:27:27.140 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:27.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:27.198 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:27.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:27.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:27.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:27.335 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:27.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:27.436 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:27.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4197091724' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:27:27.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:27 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 15257/197823 objects degraded (7.712%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:27.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4197091724' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:27:27.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:27 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 15257/197823 objects degraded (7.712%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:27.482 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:27.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:27.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4197091724' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:27:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:27 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 15257/197823 objects degraded (7.712%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:27.584 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:27.625 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:27.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:27.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:27.727 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:27.757 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:27.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:27.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:27.859 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:27.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:27.893 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:27.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:27.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:28.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:28.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:28.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:28.141 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:28.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:28.185 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:28.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:28.286 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:28.328 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:28.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:28.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:28.430 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:28 smithi032 ceph-mon[105748]: pgmap v1252: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 14983/190866 objects degraded (7.850%); 3.2 MiB/s, 50 objects/s recovering 2024-09-21T10:27:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:28 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:28 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4197091724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:27:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:28 smithi032 ceph-mon[105748]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T10:27:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:28 smithi032 ceph-mon[101299]: pgmap v1252: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 14983/190866 objects degraded (7.850%); 3.2 MiB/s, 50 objects/s recovering 2024-09-21T10:27:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:28 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:28 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4197091724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:27:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:28 smithi032 ceph-mon[101299]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T10:27:28.464 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:28.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:28.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:28 smithi045 ceph-mon[95976]: pgmap v1252: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 14983/190866 objects degraded (7.850%); 3.2 MiB/s, 50 objects/s recovering 2024-09-21T10:27:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:28 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:28 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4197091724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:27:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:28 smithi045 ceph-mon[95976]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T10:27:28.566 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:28.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:28.620 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:28.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:28.721 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:28.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:28.754 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:28.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:28.855 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:28.891 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:28.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:28.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:28.992 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:29.026 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:29.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:29.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:29.128 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:29.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:29.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:29.273 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:29.318 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:29.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:29.422 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:29.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:29.461 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:29.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:29.562 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:29.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:29.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:29.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:29.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:29.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:29.748 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:29.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:29.849 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:29.891 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:29.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:29.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:29.958 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:27:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:27:29] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-09-21T10:27:29.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:30.029 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:30.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:30.130 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:30.172 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:30.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:30.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:30.274 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:30.306 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:30.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:30.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:30.407 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:30.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:30.443 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:30.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:30.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:30 smithi032 ceph-mon[101299]: pgmap v1254: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 14983/190866 objects degraded (7.850%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T10:27:30.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:30 smithi032 ceph-mon[105748]: pgmap v1254: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 14983/190866 objects degraded (7.850%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T10:27:30.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:30 smithi045 ceph-mon[95976]: pgmap v1254: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 14983/190866 objects degraded (7.850%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T10:27:30.544 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:30.607 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:30.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:30.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:30.709 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:30.740 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:30.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:30.798 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:27:30.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:30.930 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:30.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:30.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:31.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:31.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:31.077 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:31.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:31.178 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:31.215 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:31.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:31.316 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:31.355 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:31.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:31.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:31.491 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:31.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:31.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:31.593 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:31.625 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:31.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:31.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:31.726 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:31.761 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:31.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:31.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:31.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:31.906 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:31.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:32.007 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:32.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:32.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:32.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:32.097 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:27:32.097 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:27:32.136 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:32 smithi032 ceph-mon[105748]: pgmap v1255: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 14983/185475 objects degraded (8.078%); 2.1 MiB/s, 34 objects/s recovering 2024-09-21T10:27:32.136 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:32 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14983/185475 objects degraded (8.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:32.136 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:32 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2321152442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:32.136 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:27:32.137 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:27:32.137 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:32 smithi032 ceph-mon[101299]: pgmap v1255: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 14983/185475 objects degraded (8.078%); 2.1 MiB/s, 34 objects/s recovering 2024-09-21T10:27:32.137 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:32 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14983/185475 objects degraded (8.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:32.137 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:32 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2321152442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:32.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:32.166 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:32.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:32.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:32.172 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:27:32.172 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:27:32.212 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:27:32.213 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:27:32.247 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:27:32.247 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:27:32.268 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:32.282 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:27:32.282 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:27:32.304 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:32.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:32.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:32.319 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:27:32.320 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:27:32.356 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:27:32.356 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:27:32.406 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:32.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:32.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:32 smithi045 ceph-mon[95976]: pgmap v1255: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 14983/185475 objects degraded (8.078%); 2.1 MiB/s, 34 objects/s recovering 2024-09-21T10:27:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:32 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14983/185475 objects degraded (8.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:32 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2321152442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:32.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:32.586 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:32.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:32.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:32.687 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:32.716 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:32.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:32.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:32.819 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:32.857 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:32.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:32.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:32.995 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:32.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:32.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:33.097 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:33.133 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:33.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:33.234 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:33.264 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:33.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:33.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:33.366 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:33.397 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:33.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:33.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:33.498 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:33.569 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:33.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:33.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:33.671 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:33.706 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:33.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:33.808 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:33.849 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:33.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:33.951 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:33.986 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:33.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:33.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:34.087 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:34.116 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:34.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:34.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:34 smithi032 ceph-mon[105748]: pgmap v1256: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 14697/178257 objects degraded (8.245%); 3.7 MiB/s, 58 objects/s recovering 2024-09-21T10:27:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2201890335' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:27:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:34 smithi032 ceph-mon[101299]: pgmap v1256: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 14697/178257 objects degraded (8.245%); 3.7 MiB/s, 58 objects/s recovering 2024-09-21T10:27:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:34.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:34.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2201890335' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:27:34.218 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:34.258 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:34.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:34.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:34.360 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:34.398 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:34.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:34.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:34.500 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:34 smithi045 ceph-mon[95976]: pgmap v1256: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 14697/178257 objects degraded (8.245%); 3.7 MiB/s, 58 objects/s recovering 2024-09-21T10:27:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2201890335' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:27:34.536 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:27:34.538 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:34.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:34.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:34.639 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:34.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:34.669 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:34.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:34.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:34.819 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:34.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:34.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:34.897 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:27:34.897 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:27:34.920 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:34.963 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:34.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:35.064 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:35.101 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:35.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:35.203 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:35.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:35.242 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:35.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:35.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:35.386 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:35.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:35.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:35.488 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:35.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:35.527 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:35.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:35.628 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:35.667 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:35.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:35.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:35.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:27:35.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:35.794 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:35.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:35.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:35.801 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:35 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2201890335' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:27:35.801 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:35 smithi032 ceph-mon[105748]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T10:27:35.801 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:35 smithi032 ceph-mon[105748]: pgmap v1258: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 14697/172884 objects degraded (8.501%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:27:35.802 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:35 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2201890335' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:27:35.802 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:35 smithi032 ceph-mon[101299]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T10:27:35.802 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:35 smithi032 ceph-mon[101299]: pgmap v1258: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 14697/172884 objects degraded (8.501%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:27:35.896 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:35.924 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:35.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:35.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:36.026 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:35 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2201890335' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:27:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:35 smithi045 ceph-mon[95976]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T10:27:36.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:35 smithi045 ceph-mon[95976]: pgmap v1258: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 14697/172884 objects degraded (8.501%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:27:36.055 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:36.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:36.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:36.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:36.212 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:36.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:36.314 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:36.351 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:36.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:36.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:36.452 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:36.493 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:36.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:36.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:36.596 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:36.612 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:36.643 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:27:36.643 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:27:36.660 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:36.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:36.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:36.762 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:36.805 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:36.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:36.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:36.907 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:36.940 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:36.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:36.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:37.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:37.075 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:37.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:37.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:37.208 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:37.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:37.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:37.340 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:37.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:37.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:37.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:37.490 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:37.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:37.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:37.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:37.634 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:37.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:37.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:37.736 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:37.772 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:37.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:37.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:37.874 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:37.941 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:37.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:37.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:37 smithi045 ceph-mon[95976]: pgmap v1259: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 14408/166179 objects degraded (8.670%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:27:38.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:37 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14408/166179 objects degraded (8.670%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:38.043 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:38.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:38.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:38.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:27:38.167 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:38.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:37 smithi032 ceph-mon[101299]: pgmap v1259: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 14408/166179 objects degraded (8.670%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:27:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:37 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14408/166179 objects degraded (8.670%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:37 smithi032 ceph-mon[105748]: pgmap v1259: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 14408/166179 objects degraded (8.670%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:27:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:37 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14408/166179 objects degraded (8.670%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:38.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:38.265 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:38.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:38.366 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:38.395 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:38.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:38.497 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:38.536 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:38.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:38.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:38.637 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:38.676 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:38.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:38.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:38.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:38.811 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:38.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:38.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:38.865 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:38.913 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:38.937 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:38.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:38.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:27:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:27:39.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:38 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:39.039 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:39.071 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:39.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:39.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:39.173 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:27:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:38 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:27:39.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:39.218 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:39.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:39.319 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:39.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:39.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:39.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:39.456 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:39.498 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:39.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:39.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:39.600 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:39.637 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:39.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:39.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:39.739 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:39.783 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:39.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:39.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:39.885 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:39.912 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:39.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:39.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:39 smithi032 ceph-mon[105748]: pgmap v1260: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 14408/166179 objects degraded (8.670%); 3.6 MiB/s, 57 objects/s recovering 2024-09-21T10:27:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:27:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:27:39] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.51.0" 2024-09-21T10:27:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:39 smithi032 ceph-mon[101299]: pgmap v1260: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 14408/166179 objects degraded (8.670%); 3.6 MiB/s, 57 objects/s recovering 2024-09-21T10:27:40.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:40.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:39 smithi045 ceph-mon[95976]: pgmap v1260: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 14408/166179 objects degraded (8.670%); 3.6 MiB/s, 57 objects/s recovering 2024-09-21T10:27:40.059 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:40.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:40.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:40.161 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:40.198 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:40.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:40.299 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:40.332 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:40.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:40.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:40.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:40.473 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:40.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:40.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:40.574 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:40.596 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:40.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:40.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:40.698 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:40.732 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:40.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:40.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:40.834 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:40.872 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:40.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:40.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:40.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:41.002 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:41.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:41.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:41.104 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:41.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:41.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:41.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:41.250 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:41.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:41.286 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:41.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:41.387 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:41.429 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:41.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:41.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:41.530 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:41.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:41.583 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:41.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:41.684 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:41.723 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:41.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:41.824 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:41.861 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:41.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:41.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:41.963 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:41.998 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:42.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:42.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:41 smithi045 ceph-mon[95976]: pgmap v1261: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 14408/160914 objects degraded (8.954%); 3.6 MiB/s, 57 objects/s recovering 2024-09-21T10:27:42.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:42.139 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:42.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:42.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:41 smithi032 ceph-mon[105748]: pgmap v1261: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 14408/160914 objects degraded (8.954%); 3.6 MiB/s, 57 objects/s recovering 2024-09-21T10:27:42.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:41 smithi032 ceph-mon[101299]: pgmap v1261: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 14408/160914 objects degraded (8.954%); 3.6 MiB/s, 57 objects/s recovering 2024-09-21T10:27:42.240 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:42.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:42.278 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:42.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:42.378 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:42.422 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:42.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:42.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:42.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:42.552 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:42.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:42.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:42.654 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:42.701 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:42.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:42.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:42.802 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:42.847 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:42.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:42.949 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:42.978 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:42.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:42.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:42 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14408/160914 objects degraded (8.954%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:43.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:43.082 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:42 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14408/160914 objects degraded (8.954%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:43.083 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:42 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14408/160914 objects degraded (8.954%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:43.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:43.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:43.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:43.211 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:43.249 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:43.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:43.351 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:43.389 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:43.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:43.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:43.491 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:43.531 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:43.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:43.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:43.633 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:43.668 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:43.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:43.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:43.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:43.808 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:43.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:43.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:43.910 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:43.940 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:43.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:43.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:43.976 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:44.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:44.079 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:44.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:44.106 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:43 smithi032 ceph-mon[105748]: pgmap v1262: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 14034/154119 objects degraded (9.106%); 4.1 MiB/s, 66 objects/s recovering 2024-09-21T10:27:44.106 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:43 smithi032 ceph-mon[101299]: pgmap v1262: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 14034/154119 objects degraded (9.106%); 4.1 MiB/s, 66 objects/s recovering 2024-09-21T10:27:44.181 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:44.206 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:44.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:43 smithi045 ceph-mon[95976]: pgmap v1262: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 14034/154119 objects degraded (9.106%); 4.1 MiB/s, 66 objects/s recovering 2024-09-21T10:27:44.308 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:44.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:44.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:44.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:44.456 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:44.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:44.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:44.557 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:44.579 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:44.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:44.681 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:44.706 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:44.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:44.808 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:44.831 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:44.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:44.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:44.835 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:27:44.835 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:27:44.932 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:44.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:44 smithi032 ceph-mon[105748]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T10:27:44.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:44 smithi032 ceph-mon[101299]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T10:27:44.959 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:44.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:44.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:45.061 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:45.102 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:45.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:45.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:45.204 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:45.243 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:45.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:45.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:45.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:44 smithi045 ceph-mon[95976]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T10:27:45.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:45.392 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:45.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:45.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:45.494 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:45.538 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:45.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:45.640 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:45.679 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:45.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:45.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:45.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:45.811 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:27:45.824 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:45.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:45.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:45.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:45.952 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:45.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:45.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:45.984 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:27:46.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:46.111 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:46.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:46.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:45 smithi032 ceph-mon[101299]: pgmap v1264: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 14013/148794 objects degraded (9.418%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T10:27:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:45 smithi032 ceph-mon[101299]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T10:27:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:45 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1442410061' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:27:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:45 smithi032 ceph-mon[105748]: pgmap v1264: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 14013/148794 objects degraded (9.418%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T10:27:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:45 smithi032 ceph-mon[105748]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T10:27:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:45 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1442410061' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:27:46.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:46.243 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:46.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:46.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:46.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:45 smithi045 ceph-mon[95976]: pgmap v1264: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 14013/148794 objects degraded (9.418%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T10:27:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:45 smithi045 ceph-mon[95976]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T10:27:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:45 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1442410061' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:27:46.345 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:46.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:46.380 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:46.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:46.481 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:46.514 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:46.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:46.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:46.616 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:46.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:46.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:46.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:46.757 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:46.787 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:46.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:46.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:46.914 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:46.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:46.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:47.016 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:47.051 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:47.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:47.154 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:47.160 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:46 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:47.161 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:46 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1442410061' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:27:47.161 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:46 smithi032 ceph-mon[105748]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T10:27:47.161 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:46 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 14013/148794 objects degraded (9.418%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:47.161 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:46 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:47.161 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:46 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1442410061' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:27:47.161 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:46 smithi032 ceph-mon[101299]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T10:27:47.161 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:46 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 14013/148794 objects degraded (9.418%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:47.189 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:47.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:47.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:46 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:46 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1442410061' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:27:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:46 smithi045 ceph-mon[95976]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T10:27:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:46 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 14013/148794 objects degraded (9.418%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:47.291 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:47.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:47.330 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:47.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:47.431 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:47.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:47.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:47.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:47.580 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:47.621 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:47.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:47.723 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:47.755 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:47.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:47.857 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:47.893 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:47.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:47.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:47.995 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:48.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:48.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:48.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:48.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:48.179 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:48.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:48.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:47 smithi032 ceph-mon[105748]: pgmap v1267: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 14013/143592 objects degraded (9.759%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T10:27:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:47 smithi032 ceph-mon[101299]: pgmap v1267: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 14013/143592 objects degraded (9.759%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T10:27:48.280 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:47 smithi045 ceph-mon[95976]: pgmap v1267: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 14013/143592 objects degraded (9.759%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T10:27:48.323 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:48.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:48.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:48.425 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:48.461 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:48.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:48.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:48.563 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:48.612 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:48.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:48.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:48.714 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:48.760 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:48.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:48.862 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:48.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:48.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:48.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:48.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:49.017 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:49.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:49.023 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:27:49.029 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:27:49.119 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:49.149 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:49.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:49.158 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:49.158 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:49.250 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:49.253 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:27:49.253 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:27:49.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:49.298 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:27:49.298 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:27:49.298 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:49.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:49.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:49.330 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:27:49.331 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:27:49.368 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:27:49.369 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:27:49.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:27:49.399 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:27:49.400 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:49.429 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:27:49.429 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:27:49.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:49.432 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:49.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:49.450 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:49.493 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:27:49.494 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:27:49.522 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:27:49.523 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:27:49.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:49.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:49.584 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:49.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:49.684 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:49.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:27:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:27:49] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-21T10:27:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:49.715 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:49.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:49.816 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:49.848 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:49.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:49.949 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:49.981 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:49.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:49.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:50.083 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:50.123 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:50.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:50.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:50.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:49 smithi032 ceph-mon[105748]: pgmap v1268: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 13783/142758 objects degraded (9.655%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:27:50.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:49 smithi032 ceph-mon[105748]: from='client.46037 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:27:50.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:49 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2812885798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:49 smithi032 ceph-mon[101299]: pgmap v1268: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 13783/142758 objects degraded (9.655%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:27:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:49 smithi032 ceph-mon[101299]: from='client.46037 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:27:50.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:49 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2812885798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:50.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:50.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:50.271 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:50.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:50.282 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1268,"stamp":"2024-09-21T10:27:48.641517+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3121485821,"num_objects":47586,"num_object_clones":0,"num_object_copies":142758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13783,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47586,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":208950,"num_write_kb":8191762,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4884,"num_bytes_recovered":318183045,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128316,"ondisk_log_size":128316,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14105456,"kb_used_data":8846488,"kb_used_omap":45,"kb_used_meta":5258770,"kb_avail":735855760,"statfs":{"total":767960285184,"available":753516298240,"internally_reserved":0,"allocated":9058803712,"data_stored":9042176107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46417,"internal_metadata":5384981167},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-248184832,"num_objects":-3787,"num_object_clones":0,"num_object_copies":-11361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-251,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3787,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":251,"num_bytes_recovered":16449536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002046"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":695,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.905226+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:27:43.905226+0000","last_peered":"2024-09-21T10:27:43.905226+0000","last_clean":"2024-09-21T10:27:43.905226+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:27:43.905226+0000","last_undegraded":"2024-09-21T10:27:43.905226+0000","last_fullsized":"2024-09-21T10:27:43.905226+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":413,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.909488+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:27:43.909488+0000","last_peered":"2024-09-21T10:27:43.909488+0000","last_clean":"2024-09-21T10:27:43.909488+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:27:43.909488+0000","last_undegraded":"2024-09-21T10:27:43.909488+0000","last_fullsized":"2024-09-21T10:27:43.909488+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.904659+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:27:43.904659+0000","last_peered":"2024-09-21T10:27:43.904659+0000","last_clean":"2024-09-21T10:27:43.904659+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:27:43.904659+0000","last_undegraded":"2024-09-21T10:27:43.904659+0000","last_fullsized":"2024-09-21T10:27:43.904659+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.926121+0000","last_change":"2024-09-21T10:25:48.765404+0000","last_active":"2024-09-21T10:27:45.926121+0000","last_peered":"2024-09-21T10:27:45.926121+0000","last_clean":"2024-09-21T10:27:45.926121+0000","last_became_active":"2024-09-21T10:25:48.764230+0000","last_became_peered":"2024-09-21T10:25:48.764230+0000","last_unstale":"2024-09-21T10:27:45.926121+0000","last_undegraded":"2024-09-21T10:27:45.926121+0000","last_fullsized":"2024-09-21T10:27:45.926121+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:38.551941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.926525+0000","last_change":"2024-09-21T10:25:48.764391+0000","last_active":"2024-09-21T10:27:45.926525+0000","last_peered":"2024-09-21T10:27:45.926525+0000","last_clean":"2024-09-21T10:27:45.926525+0000","last_became_active":"2024-09-21T10:25:48.763375+0000","last_became_peered":"2024-09-21T10:25:48.763375+0000","last_unstale":"2024-09-21T10:27:45.926525+0000","last_undegraded":"2024-09-21T10:27:45.926525+0000","last_fullsized":"2024-09-21T10:27:45.926525+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:03.457716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":384,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.910923+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:27:43.910923+0000","last_peered":"2024-09-21T10:27:43.910923+0000","last_clean":"2024-09-21T10:27:43.910923+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:27:43.910923+0000","last_undegraded":"2024-09-21T10:27:43.910923+0000","last_fullsized":"2024-09-21T10:27:43.910923+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":715,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.916150+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:27:43.916150+0000","last_peered":"2024-09-21T10:27:43.916150+0000","last_clean":"2024-09-21T10:27:43.916150+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:27:43.916150+0000","last_undegraded":"2024-09-21T10:27:43.916150+0000","last_fullsized":"2024-09-21T10:27:43.916150+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":480,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:39.047262+0000","last_change":"2024-09-21T10:25:48.752712+0000","last_active":"2024-09-21T10:27:39.047262+0000","last_peered":"2024-09-21T10:27:39.047262+0000","last_clean":"2024-09-21T10:27:39.047262+0000","last_became_active":"2024-09-21T10:25:48.752204+0000","last_became_peered":"2024-09-21T10:25:48.752204+0000","last_unstale":"2024-09-21T10:27:39.047262+0000","last_undegraded":"2024-09-21T10:27:39.047262+0000","last_fullsized":"2024-09-21T10:27:39.047262+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:40.068520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"410'13104","reported_seq":22326,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:47.031013+0000","last_change":"2024-09-21T10:27:44.942565+0000","last_active":"2024-09-21T10:27:47.031013+0000","last_peered":"2024-09-21T10:27:47.031013+0000","last_clean":"2024-09-21T10:27:47.031013+0000","last_became_active":"2024-09-21T10:27:44.909231+0000","last_became_peered":"2024-09-21T10:27:44.909231+0000","last_unstale":"2024-09-21T10:27:47.031013+0000","last_undegraded":"2024-09-21T10:27:47.031013+0000","last_fullsized":"2024-09-21T10:27:47.031013+0000","mapping_epoch":408,"log_start":"384'4100","ondisk_log_start":"384'4100","created":378,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":9004,"log_dups_size":0,"ondisk_log_size":9004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:13.740223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190840832,"num_objects":2912,"num_object_clones":0,"num_object_copies":8736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13104,"num_write_kb":512512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4576,"num_bytes_recovered":299892736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":114,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.542493+0000","last_change":"2024-09-21T10:25:48.755730+0000","last_active":"2024-09-21T10:27:34.542493+0000","last_peered":"2024-09-21T10:27:34.542493+0000","last_clean":"2024-09-21T10:27:34.542493+0000","last_became_active":"2024-09-21T10:25:48.755344+0000","last_became_peered":"2024-09-21T10:25:48.755344+0000","last_unstale":"2024-09-21T10:27:34.542493+0000","last_undegraded":"2024-09-21T10:27:34.542493+0000","last_fullsized":"2024-09-21T10:27:34.542493+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:37.725090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":118,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.905476+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:27:43.905476+0000","last_peered":"2024-09-21T10:27:43.905476+0000","last_clean":"2024-09-21T10:27:43.905476+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:27:43.905476+0000","last_undegraded":"2024-09-21T10:27:43.905476+0000","last_fullsized":"2024-09-21T10:27:43.905476+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"409'13006","reported_seq":13080,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.848822+0000","last_change":"2024-09-21T10:24:25.829780+0000","last_active":"2024-09-21T10:27:45.848822+0000","last_peered":"2024-09-21T10:27:45.848822+0000","last_clean":"2024-09-21T10:27:45.848822+0000","last_became_active":"2024-09-21T10:24:25.829509+0000","last_became_peered":"2024-09-21T10:24:25.829509+0000","last_unstale":"2024-09-21T10:27:45.848822+0000","last_undegraded":"2024-09-21T10:27:45.848822+0000","last_fullsized":"2024-09-21T10:27:45.848822+0000","mapping_epoch":378,"log_start":"385'5600","ondisk_log_start":"385'5600","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7406,"log_dups_size":0,"ondisk_log_size":7406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:30.108791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192806912,"num_objects":2942,"num_object_clones":0,"num_object_copies":8826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13006,"num_write_kb":510336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":86,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.542347+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:27:34.542347+0000","last_peered":"2024-09-21T10:27:34.542347+0000","last_clean":"2024-09-21T10:27:34.542347+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:27:34.542347+0000","last_undegraded":"2024-09-21T10:27:34.542347+0000","last_fullsized":"2024-09-21T10:27:34.542347+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":105,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.552805+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:27:34.552805+0000","last_peered":"2024-09-21T10:27:34.552805+0000","last_clean":"2024-09-21T10:27:34.552805+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:27:34.552805+0000","last_undegraded":"2024-09-21T10:27:34.552805+0000","last_fullsized":"2024-09-21T10:27:34.552805+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"409'13075","reported_seq":13116,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.868718+0000","last_change":"2024-09-21T10:24:25.825494+0000","last_active":"2024-09-21T10:27:45.868718+0000","last_peered":"2024-09-21T10:27:45.868718+0000","last_clean":"2024-09-21T10:27:45.868718+0000","last_became_active":"2024-09-21T10:24:25.825196+0000","last_became_peered":"2024-09-21T10:24:25.825196+0000","last_unstale":"2024-09-21T10:27:45.868718+0000","last_undegraded":"2024-09-21T10:27:45.868718+0000","last_fullsized":"2024-09-21T10:27:45.868718+0000","mapping_epoch":378,"log_start":"385'4200","ondisk_log_start":"385'4200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8875,"log_dups_size":0,"ondisk_log_size":8875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:50.985537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193265664,"num_objects":2949,"num_object_clones":0,"num_object_copies":8847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13075,"num_write_kb":512768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":408,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.542483+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:27:34.542483+0000","last_peered":"2024-09-21T10:27:34.542483+0000","last_clean":"2024-09-21T10:27:34.542483+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:27:34.542483+0000","last_undegraded":"2024-09-21T10:27:34.542483+0000","last_fullsized":"2024-09-21T10:27:34.542483+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":428,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.546664+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:27:34.546664+0000","last_peered":"2024-09-21T10:27:34.546664+0000","last_clean":"2024-09-21T10:27:34.546664+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:27:34.546664+0000","last_undegraded":"2024-09-21T10:27:34.546664+0000","last_fullsized":"2024-09-21T10:27:34.546664+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"409'12732","reported_seq":12773,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.854901+0000","last_change":"2024-09-21T10:24:25.846770+0000","last_active":"2024-09-21T10:27:45.854901+0000","last_peered":"2024-09-21T10:27:45.854901+0000","last_clean":"2024-09-21T10:27:45.854901+0000","last_became_active":"2024-09-21T10:24:25.846530+0000","last_became_peered":"2024-09-21T10:24:25.846530+0000","last_unstale":"2024-09-21T10:27:45.854901+0000","last_undegraded":"2024-09-21T10:27:45.854901+0000","last_fullsized":"2024-09-21T10:27:45.854901+0000","mapping_epoch":378,"log_start":"384'3900","ondisk_log_start":"384'3900","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8832,"log_dups_size":0,"ondisk_log_size":8832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:50.827496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187170816,"num_objects":2856,"num_object_clones":0,"num_object_copies":8568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12732,"num_write_kb":498816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":343,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.903475+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:27:43.903475+0000","last_peered":"2024-09-21T10:27:43.903475+0000","last_clean":"2024-09-21T10:27:43.903475+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:27:43.903475+0000","last_undegraded":"2024-09-21T10:27:43.903475+0000","last_fullsized":"2024-09-21T10:27:43.903475+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":256,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.927673+0000","last_change":"2024-09-21T10:25:48.763096+0000","last_active":"2024-09-21T10:27:45.927673+0000","last_peered":"2024-09-21T10:27:45.927673+0000","last_clean":"2024-09-21T10:27:45.927673+0000","last_became_active":"2024-09-21T10:25:48.761327+0000","last_became_peered":"2024-09-21T10:25:48.761327+0000","last_unstale":"2024-09-21T10:27:45.927673+0000","last_undegraded":"2024-09-21T10:27:45.927673+0000","last_fullsized":"2024-09-21T10:27:45.927673+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:23.154333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"408'13092","reported_seq":13164,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.348040+0000","last_change":"2024-09-21T10:24:25.846977+0000","last_active":"2024-09-21T10:27:44.348040+0000","last_peered":"2024-09-21T10:27:44.348040+0000","last_clean":"2024-09-21T10:27:44.348040+0000","last_became_active":"2024-09-21T10:24:25.846670+0000","last_became_peered":"2024-09-21T10:24:25.846670+0000","last_unstale":"2024-09-21T10:27:44.348040+0000","last_undegraded":"2024-09-21T10:27:44.348040+0000","last_fullsized":"2024-09-21T10:27:44.348040+0000","mapping_epoch":378,"log_start":"386'6200","ondisk_log_start":"386'6200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:28.387249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608000,"num_objects":3000,"num_object_clones":0,"num_object_copies":9000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13092,"num_write_kb":514944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":693,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.546212+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:27:34.546212+0000","last_peered":"2024-09-21T10:27:34.546212+0000","last_clean":"2024-09-21T10:27:34.546212+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:27:34.546212+0000","last_undegraded":"2024-09-21T10:27:34.546212+0000","last_fullsized":"2024-09-21T10:27:34.546212+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":756,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.904629+0000","last_change":"2024-09-21T10:25:48.755999+0000","last_active":"2024-09-21T10:27:43.904629+0000","last_peered":"2024-09-21T10:27:43.904629+0000","last_clean":"2024-09-21T10:27:43.904629+0000","last_became_active":"2024-09-21T10:25:48.754257+0000","last_became_peered":"2024-09-21T10:25:48.754257+0000","last_unstale":"2024-09-21T10:27:43.904629+0000","last_undegraded":"2024-09-21T10:27:43.904629+0000","last_fullsized":"2024-09-21T10:27:43.904629+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:06.025116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":105,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.555426+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:27:34.555426+0000","last_peered":"2024-09-21T10:27:34.555426+0000","last_clean":"2024-09-21T10:27:34.555426+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:27:34.555426+0000","last_undegraded":"2024-09-21T10:27:34.555426+0000","last_fullsized":"2024-09-21T10:27:34.555426+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"409'13153","reported_seq":13227,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.812457+0000","last_change":"2024-09-21T10:24:25.847167+0000","last_active":"2024-09-21T10:27:45.812457+0000","last_peered":"2024-09-21T10:27:45.812457+0000","last_clean":"2024-09-21T10:27:45.812457+0000","last_became_active":"2024-09-21T10:24:25.846878+0000","last_became_peered":"2024-09-21T10:24:25.846878+0000","last_unstale":"2024-09-21T10:27:45.812457+0000","last_undegraded":"2024-09-21T10:27:45.812457+0000","last_fullsized":"2024-09-21T10:27:45.812457+0000","mapping_epoch":378,"log_start":"385'5800","ondisk_log_start":"385'5800","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7353,"log_dups_size":0,"ondisk_log_size":7353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:32.840244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197722112,"num_objects":3017,"num_object_clones":0,"num_object_copies":9051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13153,"num_write_kb":517440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":467,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.905406+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:27:43.905406+0000","last_peered":"2024-09-21T10:27:43.905406+0000","last_clean":"2024-09-21T10:27:43.905406+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:27:43.905406+0000","last_undegraded":"2024-09-21T10:27:43.905406+0000","last_fullsized":"2024-09-21T10:27:43.905406+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":418,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.546749+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:27:34.546749+0000","last_peered":"2024-09-21T10:27:34.546749+0000","last_clean":"2024-09-21T10:27:34.546749+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:27:34.546749+0000","last_undegraded":"2024-09-21T10:27:34.546749+0000","last_fullsized":"2024-09-21T10:27:34.546749+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.553640+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:27:34.553640+0000","last_peered":"2024-09-21T10:27:34.553640+0000","last_clean":"2024-09-21T10:27:34.553640+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:27:34.553640+0000","last_undegraded":"2024-09-21T10:27:34.553640+0000","last_fullsized":"2024-09-21T10:27:34.553640+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"408'13043","reported_seq":13083,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.175906+0000","last_change":"2024-09-21T10:24:25.828673+0000","last_active":"2024-09-21T10:27:44.175906+0000","last_peered":"2024-09-21T10:27:44.175906+0000","last_clean":"2024-09-21T10:27:44.175906+0000","last_became_active":"2024-09-21T10:24:25.828433+0000","last_became_peered":"2024-09-21T10:24:25.828433+0000","last_unstale":"2024-09-21T10:27:44.175906+0000","last_undegraded":"2024-09-21T10:27:44.175906+0000","last_fullsized":"2024-09-21T10:27:44.175906+0000","mapping_epoch":378,"log_start":"386'6000","ondisk_log_start":"386'6000","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7043,"log_dups_size":0,"ondisk_log_size":7043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:07.654019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195100672,"num_objects":2977,"num_object_clones":0,"num_object_copies":8931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13043,"num_write_kb":512640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":105,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.552654+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:27:34.552654+0000","last_peered":"2024-09-21T10:27:34.552654+0000","last_clean":"2024-09-21T10:27:34.552654+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:27:34.552654+0000","last_undegraded":"2024-09-21T10:27:34.552654+0000","last_fullsized":"2024-09-21T10:27:34.552654+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":105,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.554179+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:27:34.554179+0000","last_peered":"2024-09-21T10:27:34.554179+0000","last_clean":"2024-09-21T10:27:34.554179+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:27:34.554179+0000","last_undegraded":"2024-09-21T10:27:34.554179+0000","last_fullsized":"2024-09-21T10:27:34.554179+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"408'13095","reported_seq":13167,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.319510+0000","last_change":"2024-09-21T10:24:25.847020+0000","last_active":"2024-09-21T10:27:44.319510+0000","last_peered":"2024-09-21T10:27:44.319510+0000","last_clean":"2024-09-21T10:27:44.319510+0000","last_became_active":"2024-09-21T10:24:25.846622+0000","last_became_peered":"2024-09-21T10:24:25.846622+0000","last_unstale":"2024-09-21T10:27:44.319510+0000","last_undegraded":"2024-09-21T10:27:44.319510+0000","last_fullsized":"2024-09-21T10:27:44.319510+0000","mapping_epoch":378,"log_start":"386'6200","ondisk_log_start":"386'6200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6895,"log_dups_size":0,"ondisk_log_size":6895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:54:04.668262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195362816,"num_objects":2981,"num_object_clones":0,"num_object_copies":8943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13095,"num_write_kb":514432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":645,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.903216+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:27:43.903216+0000","last_peered":"2024-09-21T10:27:43.903216+0000","last_clean":"2024-09-21T10:27:43.903216+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:27:43.903216+0000","last_undegraded":"2024-09-21T10:27:43.903216+0000","last_fullsized":"2024-09-21T10:27:43.903216+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":724,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.909472+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:27:43.909472+0000","last_peered":"2024-09-21T10:27:43.909472+0000","last_clean":"2024-09-21T10:27:43.909472+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:27:43.909472+0000","last_undegraded":"2024-09-21T10:27:43.909472+0000","last_fullsized":"2024-09-21T10:27:43.909472+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":418,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.986553+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:27:44.986553+0000","last_peered":"2024-09-21T10:27:44.986553+0000","last_clean":"2024-09-21T10:27:44.986553+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:27:44.986553+0000","last_undegraded":"2024-09-21T10:27:44.986553+0000","last_fullsized":"2024-09-21T10:27:44.986553+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"408'12909","reported_seq":13036,"reported_epoch":408,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:27:44.389832+0000","last_change":"2024-09-21T10:27:43.773504+0000","last_active":"2024-09-21T10:27:44.389832+0000","last_peered":"2024-09-21T10:27:44.389832+0000","last_clean":"2024-09-21T10:24:52.081220+0000","last_became_active":"2024-09-21T10:25:50.482809+0000","last_became_peered":"2024-09-21T10:25:50.482809+0000","last_unstale":"2024-09-21T10:27:44.389832+0000","last_undegraded":"2024-09-21T10:25:49.768122+0000","last_fullsized":"2024-09-21T10:25:49.744590+0000","mapping_epoch":390,"log_start":"386'6000","ondisk_log_start":"386'6000","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6909,"log_dups_size":0,"ondisk_log_size":6909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195231744,"num_objects":2979,"num_object_clones":0,"num_object_copies":8937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4514,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12909,"num_write_kb":508416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":1441792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-1535}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":693,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.910307+0000","last_change":"2024-09-21T10:25:48.754725+0000","last_active":"2024-09-21T10:27:43.910307+0000","last_peered":"2024-09-21T10:27:43.910307+0000","last_clean":"2024-09-21T10:27:43.910307+0000","last_became_active":"2024-09-21T10:25:48.754341+0000","last_became_peered":"2024-09-21T10:25:48.754341+0000","last_unstale":"2024-09-21T10:27:43.910307+0000","last_undegraded":"2024-09-21T10:27:43.910307+0000","last_fullsized":"2024-09-21T10:27:43.910307+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:25.191555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:25.191555+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:08.981054+0000","scrub_duration":0.0015044590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":722,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.905698+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:27:43.905698+0000","last_peered":"2024-09-21T10:27:43.905698+0000","last_clean":"2024-09-21T10:27:43.905698+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:27:43.905698+0000","last_undegraded":"2024-09-21T10:27:43.905698+0000","last_fullsized":"2024-09-21T10:27:43.905698+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":655,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.908035+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:27:43.908035+0000","last_peered":"2024-09-21T10:27:43.908035+0000","last_clean":"2024-09-21T10:27:43.908035+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:27:43.908035+0000","last_undegraded":"2024-09-21T10:27:43.908035+0000","last_fullsized":"2024-09-21T10:27:43.908035+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":384,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.547287+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:27:34.547287+0000","last_peered":"2024-09-21T10:27:34.547287+0000","last_clean":"2024-09-21T10:27:34.547287+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:27:34.547287+0000","last_undegraded":"2024-09-21T10:27:34.547287+0000","last_fullsized":"2024-09-21T10:27:34.547287+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"409'12808","reported_seq":12864,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:27:45.904208+0000","last_change":"2024-09-21T10:25:50.516299+0000","last_active":"2024-09-21T10:27:45.904208+0000","last_peered":"2024-09-21T10:27:45.904208+0000","last_clean":"2024-09-21T10:24:52.080442+0000","last_became_active":"2024-09-21T10:25:50.497794+0000","last_became_peered":"2024-09-21T10:25:50.497794+0000","last_unstale":"2024-09-21T10:27:45.904208+0000","last_undegraded":"2024-09-21T10:25:49.773994+0000","last_fullsized":"2024-09-21T10:25:49.744196+0000","mapping_epoch":390,"log_start":"384'3900","ondisk_log_start":"384'3900","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8908,"log_dups_size":0,"ondisk_log_size":8908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:02.445442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191496192,"num_objects":2922,"num_object_clones":0,"num_object_copies":8766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4583,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12808,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":-1661}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":674,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.985258+0000","last_change":"2024-09-21T10:24:27.359111+0000","last_active":"2024-09-21T10:27:44.985258+0000","last_peered":"2024-09-21T10:27:44.985258+0000","last_clean":"2024-09-21T10:27:44.985258+0000","last_became_active":"2024-09-21T10:24:24.825850+0000","last_became_peered":"2024-09-21T10:24:24.825850+0000","last_unstale":"2024-09-21T10:27:44.985258+0000","last_undegraded":"2024-09-21T10:27:44.985258+0000","last_fullsized":"2024-09-21T10:27:44.985258+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:27.359065+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:24:27.359065+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:06.871319+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1065,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.541618+0000","last_change":"2024-09-21T10:25:48.755667+0000","last_active":"2024-09-21T10:27:34.541618+0000","last_peered":"2024-09-21T10:27:34.541618+0000","last_clean":"2024-09-21T10:27:34.541618+0000","last_became_active":"2024-09-21T10:25:48.755060+0000","last_became_peered":"2024-09-21T10:25:48.755060+0000","last_unstale":"2024-09-21T10:27:34.541618+0000","last_undegraded":"2024-09-21T10:27:34.541618+0000","last_fullsized":"2024-09-21T10:27:34.541618+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:09.230030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":258,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.927048+0000","last_change":"2024-09-21T10:25:48.754659+0000","last_active":"2024-09-21T10:27:45.927048+0000","last_peered":"2024-09-21T10:27:45.927048+0000","last_clean":"2024-09-21T10:27:45.927048+0000","last_became_active":"2024-09-21T10:25:48.754230+0000","last_became_peered":"2024-09-21T10:25:48.754230+0000","last_unstale":"2024-09-21T10:27:45.927048+0000","last_undegraded":"2024-09-21T10:27:45.927048+0000","last_fullsized":"2024-09-21T10:27:45.927048+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:12.555868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":65,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.926315+0000","last_change":"2024-09-21T10:25:48.761249+0000","last_active":"2024-09-21T10:27:45.926315+0000","last_peered":"2024-09-21T10:27:45.926315+0000","last_clean":"2024-09-21T10:27:45.926315+0000","last_became_active":"2024-09-21T10:25:48.759359+0000","last_became_peered":"2024-09-21T10:25:48.759359+0000","last_unstale":"2024-09-21T10:27:45.926315+0000","last_undegraded":"2024-09-21T10:27:45.926315+0000","last_fullsized":"2024-09-21T10:27:45.926315+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:33.510936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"409'13011","reported_seq":13052,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.849489+0000","last_change":"2024-09-21T10:24:25.827798+0000","last_active":"2024-09-21T10:27:45.849489+0000","last_peered":"2024-09-21T10:27:45.849489+0000","last_clean":"2024-09-21T10:27:45.849489+0000","last_became_active":"2024-09-21T10:24:25.827230+0000","last_became_peered":"2024-09-21T10:24:25.827230+0000","last_unstale":"2024-09-21T10:27:45.849489+0000","last_undegraded":"2024-09-21T10:27:45.849489+0000","last_fullsized":"2024-09-21T10:27:45.849489+0000","mapping_epoch":378,"log_start":"384'4100","ondisk_log_start":"384'4100","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8911,"log_dups_size":0,"ondisk_log_size":8911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:28:46.430403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199294976,"num_objects":3041,"num_object_clones":0,"num_object_copies":9123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13011,"num_write_kb":513664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":708,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.545497+0000","last_change":"2024-09-21T10:25:48.755509+0000","last_active":"2024-09-21T10:27:34.545497+0000","last_peered":"2024-09-21T10:27:34.545497+0000","last_clean":"2024-09-21T10:27:34.545497+0000","last_became_active":"2024-09-21T10:25:48.754558+0000","last_became_peered":"2024-09-21T10:25:48.754558+0000","last_unstale":"2024-09-21T10:27:34.545497+0000","last_undegraded":"2024-09-21T10:27:34.545497+0000","last_fullsized":"2024-09-21T10:27:34.545497+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:11.262503+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1255,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.912041+0000","last_change":"2024-09-21T10:25:48.753359+0000","last_active":"2024-09-21T10:27:43.912041+0000","last_peered":"2024-09-21T10:27:43.912041+0000","last_clean":"2024-09-21T10:27:43.912041+0000","last_became_active":"2024-09-21T10:25:48.752653+0000","last_became_peered":"2024-09-21T10:25:48.752653+0000","last_unstale":"2024-09-21T10:27:43.912041+0000","last_undegraded":"2024-09-21T10:27:43.912041+0000","last_fullsized":"2024-09-21T10:27:43.912041+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:55.095131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":405,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.985074+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:27:44.985074+0000","last_peered":"2024-09-21T10:27:44.985074+0000","last_clean":"2024-09-21T10:27:44.985074+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:27:44.985074+0000","last_undegraded":"2024-09-21T10:27:44.985074+0000","last_fullsized":"2024-09-21T10:27:44.985074+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":445,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.986635+0000","last_change":"2024-09-21T10:25:48.766755+0000","last_active":"2024-09-21T10:27:44.986635+0000","last_peered":"2024-09-21T10:27:44.986635+0000","last_clean":"2024-09-21T10:27:44.986635+0000","last_became_active":"2024-09-21T10:25:48.766298+0000","last_became_peered":"2024-09-21T10:25:48.766298+0000","last_unstale":"2024-09-21T10:27:44.986635+0000","last_undegraded":"2024-09-21T10:27:44.986635+0000","last_fullsized":"2024-09-21T10:27:44.986635+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:18.788447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"409'13151","reported_seq":13210,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:27:45.863179+0000","last_change":"2024-09-21T10:25:50.515832+0000","last_active":"2024-09-21T10:27:45.863179+0000","last_peered":"2024-09-21T10:27:45.863179+0000","last_clean":"2024-09-21T10:24:52.091731+0000","last_became_active":"2024-09-21T10:25:50.504214+0000","last_became_peered":"2024-09-21T10:25:50.504214+0000","last_unstale":"2024-09-21T10:27:45.863179+0000","last_undegraded":"2024-09-21T10:25:49.774199+0000","last_fullsized":"2024-09-21T10:25:49.743948+0000","mapping_epoch":390,"log_start":"385'4300","ondisk_log_start":"385'4300","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8851,"log_dups_size":0,"ondisk_log_size":8851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:49.426271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195756032,"num_objects":2987,"num_object_clones":0,"num_object_copies":8961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13151,"num_write_kb":516416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":-1699}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":660,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.904117+0000","last_change":"2024-09-21T10:25:48.757672+0000","last_active":"2024-09-21T10:27:43.904117+0000","last_peered":"2024-09-21T10:27:43.904117+0000","last_clean":"2024-09-21T10:27:43.904117+0000","last_became_active":"2024-09-21T10:25:48.757296+0000","last_became_peered":"2024-09-21T10:25:48.757296+0000","last_unstale":"2024-09-21T10:27:43.904117+0000","last_undegraded":"2024-09-21T10:27:43.904117+0000","last_fullsized":"2024-09-21T10:27:43.904117+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:31.244809+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:31.244809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:32.654430+0000","scrub_duration":0.0015828649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1153,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.903534+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:27:43.903534+0000","last_peered":"2024-09-21T10:27:43.903534+0000","last_clean":"2024-09-21T10:27:43.903534+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:27:43.903534+0000","last_undegraded":"2024-09-21T10:27:43.903534+0000","last_fullsized":"2024-09-21T10:27:43.903534+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":120,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.553279+0000","last_change":"2024-09-21T10:25:48.753400+0000","last_active":"2024-09-21T10:27:34.553279+0000","last_peered":"2024-09-21T10:27:34.553279+0000","last_clean":"2024-09-21T10:27:34.553279+0000","last_became_active":"2024-09-21T10:25:48.753088+0000","last_became_peered":"2024-09-21T10:25:48.753088+0000","last_unstale":"2024-09-21T10:27:34.553279+0000","last_undegraded":"2024-09-21T10:27:34.553279+0000","last_fullsized":"2024-09-21T10:27:34.553279+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:49.107566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":405,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.985538+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:27:44.985538+0000","last_peered":"2024-09-21T10:27:44.985538+0000","last_clean":"2024-09-21T10:27:44.985538+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:27:44.985538+0000","last_undegraded":"2024-09-21T10:27:44.985538+0000","last_fullsized":"2024-09-21T10:27:44.985538+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"409'12954","reported_seq":12995,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.885771+0000","last_change":"2024-09-21T10:24:25.831730+0000","last_active":"2024-09-21T10:27:45.885771+0000","last_peered":"2024-09-21T10:27:45.885771+0000","last_clean":"2024-09-21T10:27:45.885771+0000","last_became_active":"2024-09-21T10:24:25.831416+0000","last_became_peered":"2024-09-21T10:24:25.831416+0000","last_unstale":"2024-09-21T10:27:45.885771+0000","last_undegraded":"2024-09-21T10:27:45.885771+0000","last_fullsized":"2024-09-21T10:27:45.885771+0000","mapping_epoch":378,"log_start":"384'4100","ondisk_log_start":"384'4100","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8854,"log_dups_size":0,"ondisk_log_size":8854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.763992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199098368,"num_objects":3038,"num_object_clones":0,"num_object_copies":9114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12954,"num_write_kb":511744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":492,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.547021+0000","last_change":"2024-09-21T10:24:24.828950+0000","last_active":"2024-09-21T10:27:34.547021+0000","last_peered":"2024-09-21T10:27:34.547021+0000","last_clean":"2024-09-21T10:27:34.547021+0000","last_became_active":"2024-09-21T10:24:24.828550+0000","last_became_peered":"2024-09-21T10:24:24.828550+0000","last_unstale":"2024-09-21T10:27:34.547021+0000","last_undegraded":"2024-09-21T10:27:34.547021+0000","last_fullsized":"2024-09-21T10:27:34.547021+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:29.313198+0000","scrub_duration":0.001370488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":358,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.904489+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:27:43.904489+0000","last_peered":"2024-09-21T10:27:43.904489+0000","last_clean":"2024-09-21T10:27:43.904489+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:27:43.904489+0000","last_undegraded":"2024-09-21T10:27:43.904489+0000","last_fullsized":"2024-09-21T10:27:43.904489+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":1024,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.985681+0000","last_change":"2024-09-21T10:25:48.760586+0000","last_active":"2024-09-21T10:27:44.985681+0000","last_peered":"2024-09-21T10:27:44.985681+0000","last_clean":"2024-09-21T10:27:44.985681+0000","last_became_active":"2024-09-21T10:25:48.760042+0000","last_became_peered":"2024-09-21T10:25:48.760042+0000","last_unstale":"2024-09-21T10:27:44.985681+0000","last_undegraded":"2024-09-21T10:27:44.985681+0000","last_fullsized":"2024-09-21T10:27:44.985681+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:41.526062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":441,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.988876+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:27:44.988876+0000","last_peered":"2024-09-21T10:27:44.988876+0000","last_clean":"2024-09-21T10:27:44.988876+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:27:44.988876+0000","last_undegraded":"2024-09-21T10:27:44.988876+0000","last_fullsized":"2024-09-21T10:27:44.988876+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"408'12874","reported_seq":12914,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.158470+0000","last_change":"2024-09-21T10:24:25.831693+0000","last_active":"2024-09-21T10:27:44.158470+0000","last_peered":"2024-09-21T10:27:44.158470+0000","last_clean":"2024-09-21T10:27:44.158470+0000","last_became_active":"2024-09-21T10:24:25.831245+0000","last_became_peered":"2024-09-21T10:24:25.831245+0000","last_unstale":"2024-09-21T10:27:44.158470+0000","last_undegraded":"2024-09-21T10:27:44.158470+0000","last_fullsized":"2024-09-21T10:27:44.158470+0000","mapping_epoch":378,"log_start":"385'5800","ondisk_log_start":"385'5800","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:30.943590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197394432,"num_objects":3012,"num_object_clones":0,"num_object_copies":9036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12874,"num_write_kb":508352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":507,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.547487+0000","last_change":"2024-09-21T10:25:48.752455+0000","last_active":"2024-09-21T10:27:34.547487+0000","last_peered":"2024-09-21T10:27:34.547487+0000","last_clean":"2024-09-21T10:27:34.547487+0000","last_became_active":"2024-09-21T10:25:48.752056+0000","last_became_peered":"2024-09-21T10:25:48.752056+0000","last_unstale":"2024-09-21T10:27:34.547487+0000","last_undegraded":"2024-09-21T10:27:34.547487+0000","last_fullsized":"2024-09-21T10:27:34.547487+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:15.886186+0000","scrub_duration":0.0018106349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":423,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.985645+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:27:44.985645+0000","last_peered":"2024-09-21T10:27:44.985645+0000","last_clean":"2024-09-21T10:27:44.985645+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:27:44.985645+0000","last_undegraded":"2024-09-21T10:27:44.985645+0000","last_fullsized":"2024-09-21T10:27:44.985645+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":991,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.542594+0000","last_change":"2024-09-21T10:25:48.755967+0000","last_active":"2024-09-21T10:27:34.542594+0000","last_peered":"2024-09-21T10:27:34.542594+0000","last_clean":"2024-09-21T10:27:34.542594+0000","last_became_active":"2024-09-21T10:25:48.755037+0000","last_became_peered":"2024-09-21T10:25:48.755037+0000","last_unstale":"2024-09-21T10:27:34.542594+0000","last_undegraded":"2024-09-21T10:27:34.542594+0000","last_fullsized":"2024-09-21T10:27:34.542594+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:26.414898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":273,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.927379+0000","last_change":"2024-09-21T10:25:48.767902+0000","last_active":"2024-09-21T10:27:45.927379+0000","last_peered":"2024-09-21T10:27:45.927379+0000","last_clean":"2024-09-21T10:27:45.927379+0000","last_became_active":"2024-09-21T10:25:48.765443+0000","last_became_peered":"2024-09-21T10:25:48.765443+0000","last_unstale":"2024-09-21T10:27:45.927379+0000","last_undegraded":"2024-09-21T10:27:45.927379+0000","last_fullsized":"2024-09-21T10:27:45.927379+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:26.622825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"408'12998","reported_seq":13039,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.114184+0000","last_change":"2024-09-21T10:24:25.827432+0000","last_active":"2024-09-21T10:27:44.114184+0000","last_peered":"2024-09-21T10:27:44.114184+0000","last_clean":"2024-09-21T10:27:44.114184+0000","last_became_active":"2024-09-21T10:24:25.827047+0000","last_became_peered":"2024-09-21T10:24:25.827047+0000","last_unstale":"2024-09-21T10:27:44.114184+0000","last_undegraded":"2024-09-21T10:27:44.114184+0000","last_fullsized":"2024-09-21T10:27:44.114184+0000","mapping_epoch":378,"log_start":"386'6000","ondisk_log_start":"386'6000","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6998,"log_dups_size":0,"ondisk_log_size":6998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:52.646714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198901784,"num_objects":3036,"num_object_clones":0,"num_object_copies":9108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3036,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12998,"num_write_kb":513025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":644,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.553711+0000","last_change":"2024-09-21T10:24:26.255840+0000","last_active":"2024-09-21T10:27:34.553711+0000","last_peered":"2024-09-21T10:27:34.553711+0000","last_clean":"2024-09-21T10:27:34.553711+0000","last_became_active":"2024-09-21T10:24:24.825549+0000","last_became_peered":"2024-09-21T10:24:24.825549+0000","last_unstale":"2024-09-21T10:27:34.553711+0000","last_undegraded":"2024-09-21T10:27:34.553711+0000","last_fullsized":"2024-09-21T10:27:34.553711+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:26.255753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.255753+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:25.638856+0000","scrub_duration":0.0016223839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":740,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.916355+0000","last_change":"2024-09-21T10:25:48.758590+0000","last_active":"2024-09-21T10:27:43.916355+0000","last_peered":"2024-09-21T10:27:43.916355+0000","last_clean":"2024-09-21T10:27:43.916355+0000","last_became_active":"2024-09-21T10:25:48.758189+0000","last_became_peered":"2024-09-21T10:25:48.758189+0000","last_unstale":"2024-09-21T10:27:43.916355+0000","last_undegraded":"2024-09-21T10:27:43.916355+0000","last_fullsized":"2024-09-21T10:27:43.916355+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:24.126237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1076,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.986391+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:27:44.986391+0000","last_peered":"2024-09-21T10:27:44.986391+0000","last_clean":"2024-09-21T10:27:44.986391+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:27:44.986391+0000","last_undegraded":"2024-09-21T10:27:44.986391+0000","last_fullsized":"2024-09-21T10:27:44.986391+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":735,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.908298+0000","last_change":"2024-09-21T10:25:48.754966+0000","last_active":"2024-09-21T10:27:43.908298+0000","last_peered":"2024-09-21T10:27:43.908298+0000","last_clean":"2024-09-21T10:27:43.908298+0000","last_became_active":"2024-09-21T10:25:48.754152+0000","last_became_peered":"2024-09-21T10:25:48.754152+0000","last_unstale":"2024-09-21T10:27:43.908298+0000","last_undegraded":"2024-09-21T10:27:43.908298+0000","last_fullsized":"2024-09-21T10:27:43.908298+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:33:28.795713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"408'13037","reported_seq":13109,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.312875+0000","last_change":"2024-09-21T10:24:25.830908+0000","last_active":"2024-09-21T10:27:44.312875+0000","last_peered":"2024-09-21T10:27:44.312875+0000","last_clean":"2024-09-21T10:27:44.312875+0000","last_became_active":"2024-09-21T10:24:25.830171+0000","last_became_peered":"2024-09-21T10:24:25.830171+0000","last_unstale":"2024-09-21T10:27:44.312875+0000","last_undegraded":"2024-09-21T10:27:44.312875+0000","last_fullsized":"2024-09-21T10:27:44.312875+0000","mapping_epoch":378,"log_start":"386'6200","ondisk_log_start":"386'6200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6837,"log_dups_size":0,"ondisk_log_size":6837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:00.292732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190644224,"num_objects":2909,"num_object_clones":0,"num_object_copies":8727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13037,"num_write_kb":510272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":708,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.905038+0000","last_change":"2024-09-21T10:24:26.926717+0000","last_active":"2024-09-21T10:27:43.905038+0000","last_peered":"2024-09-21T10:27:43.905038+0000","last_clean":"2024-09-21T10:27:43.905038+0000","last_became_active":"2024-09-21T10:24:24.828205+0000","last_became_peered":"2024-09-21T10:24:24.828205+0000","last_unstale":"2024-09-21T10:27:43.905038+0000","last_undegraded":"2024-09-21T10:27:43.905038+0000","last_fullsized":"2024-09-21T10:27:43.905038+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:26.926661+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.926661+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:47.946104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":94,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.926601+0000","last_change":"2024-09-21T10:25:48.762565+0000","last_active":"2024-09-21T10:27:45.926601+0000","last_peered":"2024-09-21T10:27:45.926601+0000","last_clean":"2024-09-21T10:27:45.926601+0000","last_became_active":"2024-09-21T10:25:48.761324+0000","last_became_peered":"2024-09-21T10:25:48.761324+0000","last_unstale":"2024-09-21T10:27:45.926601+0000","last_undegraded":"2024-09-21T10:27:45.926601+0000","last_fullsized":"2024-09-21T10:27:45.926601+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:58.885790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":719,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.547806+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:27:34.547806+0000","last_peered":"2024-09-21T10:27:34.547806+0000","last_clean":"2024-09-21T10:27:34.547806+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:27:34.547806+0000","last_undegraded":"2024-09-21T10:27:34.547806+0000","last_fullsized":"2024-09-21T10:27:34.547806+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":381,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.552990+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:27:34.552990+0000","last_peered":"2024-09-21T10:27:34.552990+0000","last_clean":"2024-09-21T10:27:34.552990+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:27:34.552990+0000","last_undegraded":"2024-09-21T10:27:34.552990+0000","last_fullsized":"2024-09-21T10:27:34.552990+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":405,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.553366+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:27:34.553366+0000","last_peered":"2024-09-21T10:27:34.553366+0000","last_clean":"2024-09-21T10:27:34.553366+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:27:34.553366+0000","last_undegraded":"2024-09-21T10:27:34.553366+0000","last_fullsized":"2024-09-21T10:27:34.553366+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":115,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.555491+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:27:34.555491+0000","last_peered":"2024-09-21T10:27:34.555491+0000","last_clean":"2024-09-21T10:27:34.555491+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:27:34.555491+0000","last_undegraded":"2024-09-21T10:27:34.555491+0000","last_fullsized":"2024-09-21T10:27:34.555491+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":110,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.903458+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:27:43.903458+0000","last_peered":"2024-09-21T10:27:43.903458+0000","last_clean":"2024-09-21T10:27:43.903458+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:27:43.903458+0000","last_undegraded":"2024-09-21T10:27:43.903458+0000","last_fullsized":"2024-09-21T10:27:43.903458+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":88,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.542140+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:27:34.542140+0000","last_peered":"2024-09-21T10:27:34.542140+0000","last_clean":"2024-09-21T10:27:34.542140+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:27:34.542140+0000","last_undegraded":"2024-09-21T10:27:34.542140+0000","last_fullsized":"2024-09-21T10:27:34.542140+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":118,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.904488+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:27:43.904488+0000","last_peered":"2024-09-21T10:27:43.904488+0000","last_clean":"2024-09-21T10:27:43.904488+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:27:43.904488+0000","last_undegraded":"2024-09-21T10:27:43.904488+0000","last_fullsized":"2024-09-21T10:27:43.904488+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":648,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.904919+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:27:43.904919+0000","last_peered":"2024-09-21T10:27:43.904919+0000","last_clean":"2024-09-21T10:27:43.904919+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:27:43.904919+0000","last_undegraded":"2024-09-21T10:27:43.904919+0000","last_fullsized":"2024-09-21T10:27:43.904919+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":271,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.926945+0000","last_change":"2024-09-21T10:25:48.765786+0000","last_active":"2024-09-21T10:27:45.926945+0000","last_peered":"2024-09-21T10:27:45.926945+0000","last_clean":"2024-09-21T10:27:45.926945+0000","last_became_active":"2024-09-21T10:25:48.764011+0000","last_became_peered":"2024-09-21T10:25:48.764011+0000","last_unstale":"2024-09-21T10:27:45.926945+0000","last_undegraded":"2024-09-21T10:27:45.926945+0000","last_fullsized":"2024-09-21T10:27:45.926945+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:13.352641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":708,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.908215+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:27:43.908215+0000","last_peered":"2024-09-21T10:27:43.908215+0000","last_clean":"2024-09-21T10:27:43.908215+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:27:43.908215+0000","last_undegraded":"2024-09-21T10:27:43.908215+0000","last_fullsized":"2024-09-21T10:27:43.908215+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":423,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.547095+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:27:34.547095+0000","last_peered":"2024-09-21T10:27:34.547095+0000","last_clean":"2024-09-21T10:27:34.547095+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:27:34.547095+0000","last_undegraded":"2024-09-21T10:27:34.547095+0000","last_fullsized":"2024-09-21T10:27:34.547095+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":735,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.909329+0000","last_change":"2024-09-21T10:25:48.755341+0000","last_active":"2024-09-21T10:27:43.909329+0000","last_peered":"2024-09-21T10:27:43.909329+0000","last_clean":"2024-09-21T10:27:43.909329+0000","last_became_active":"2024-09-21T10:25:48.754620+0000","last_became_peered":"2024-09-21T10:25:48.754620+0000","last_unstale":"2024-09-21T10:27:43.909329+0000","last_undegraded":"2024-09-21T10:27:43.909329+0000","last_fullsized":"2024-09-21T10:27:43.909329+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:57:21.296549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":304,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.926400+0000","last_change":"2024-09-21T10:25:48.765683+0000","last_active":"2024-09-21T10:27:45.926400+0000","last_peered":"2024-09-21T10:27:45.926400+0000","last_clean":"2024-09-21T10:27:45.926400+0000","last_became_active":"2024-09-21T10:25:48.765281+0000","last_became_peered":"2024-09-21T10:25:48.765281+0000","last_unstale":"2024-09-21T10:27:45.926400+0000","last_undegraded":"2024-09-21T10:27:45.926400+0000","last_fullsized":"2024-09-21T10:27:45.926400+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:41.798683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":425,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.984973+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:27:44.984973+0000","last_peered":"2024-09-21T10:27:44.984973+0000","last_clean":"2024-09-21T10:27:44.984973+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:27:44.984973+0000","last_undegraded":"2024-09-21T10:27:44.984973+0000","last_fullsized":"2024-09-21T10:27:44.984973+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":714,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.905610+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:27:43.905610+0000","last_peered":"2024-09-21T10:27:43.905610+0000","last_clean":"2024-09-21T10:27:43.905610+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:27:43.905610+0000","last_undegraded":"2024-09-21T10:27:43.905610+0000","last_fullsized":"2024-09-21T10:27:43.905610+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":412,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.554061+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:27:34.554061+0000","last_peered":"2024-09-21T10:27:34.554061+0000","last_clean":"2024-09-21T10:27:34.554061+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:27:34.554061+0000","last_undegraded":"2024-09-21T10:27:34.554061+0000","last_fullsized":"2024-09-21T10:27:34.554061+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":713,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.912063+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:27:43.912063+0000","last_peered":"2024-09-21T10:27:43.912063+0000","last_clean":"2024-09-21T10:27:43.912063+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:27:43.912063+0000","last_undegraded":"2024-09-21T10:27:43.912063+0000","last_fullsized":"2024-09-21T10:27:43.912063+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":64,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.927520+0000","last_change":"2024-09-21T10:25:48.767385+0000","last_active":"2024-09-21T10:27:45.927520+0000","last_peered":"2024-09-21T10:27:45.927520+0000","last_clean":"2024-09-21T10:27:45.927520+0000","last_became_active":"2024-09-21T10:25:48.764077+0000","last_became_peered":"2024-09-21T10:25:48.764077+0000","last_unstale":"2024-09-21T10:27:45.927520+0000","last_undegraded":"2024-09-21T10:27:45.927520+0000","last_fullsized":"2024-09-21T10:27:45.927520+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:06:51.969987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":105,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.553150+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:27:34.553150+0000","last_peered":"2024-09-21T10:27:34.553150+0000","last_clean":"2024-09-21T10:27:34.553150+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:27:34.553150+0000","last_undegraded":"2024-09-21T10:27:34.553150+0000","last_fullsized":"2024-09-21T10:27:34.553150+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":350,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.903308+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:27:43.903308+0000","last_peered":"2024-09-21T10:27:43.903308+0000","last_clean":"2024-09-21T10:27:43.903308+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:27:43.903308+0000","last_undegraded":"2024-09-21T10:27:43.903308+0000","last_fullsized":"2024-09-21T10:27:43.903308+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.553936+0000","last_change":"2024-09-21T10:25:48.753233+0000","last_active":"2024-09-21T10:27:34.553936+0000","last_peered":"2024-09-21T10:27:34.553936+0000","last_clean":"2024-09-21T10:27:34.553936+0000","last_became_active":"2024-09-21T10:25:48.752932+0000","last_became_peered":"2024-09-21T10:25:48.752932+0000","last_unstale":"2024-09-21T10:27:34.553936+0000","last_undegraded":"2024-09-21T10:27:34.553936+0000","last_fullsized":"2024-09-21T10:27:34.553936+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:49.456475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":328,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.904589+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:27:43.904589+0000","last_peered":"2024-09-21T10:27:43.904589+0000","last_clean":"2024-09-21T10:27:43.904589+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:27:43.904589+0000","last_undegraded":"2024-09-21T10:27:43.904589+0000","last_fullsized":"2024-09-21T10:27:43.904589+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":738,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.905213+0000","last_change":"2024-09-21T10:25:48.755262+0000","last_active":"2024-09-21T10:27:43.905213+0000","last_peered":"2024-09-21T10:27:43.905213+0000","last_clean":"2024-09-21T10:27:43.905213+0000","last_became_active":"2024-09-21T10:25:48.754746+0000","last_became_peered":"2024-09-21T10:25:48.754746+0000","last_unstale":"2024-09-21T10:27:43.905213+0000","last_undegraded":"2024-09-21T10:27:43.905213+0000","last_fullsized":"2024-09-21T10:27:43.905213+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:28:17.312418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":689,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-09-21T10:27:43.904830+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:27:43.904830+0000","last_peered":"2024-09-21T10:27:43.904830+0000","last_clean":"2024-09-21T10:27:43.904830+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:27:43.904830+0000","last_undegraded":"2024-09-21T10:27:43.904830+0000","last_fullsized":"2024-09-21T10:27:43.904830+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":105,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-09-21T10:27:34.552647+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:27:34.552647+0000","last_peered":"2024-09-21T10:27:34.552647+0000","last_clean":"2024-09-21T10:27:34.552647+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:27:34.552647+0000","last_undegraded":"2024-09-21T10:27:34.552647+0000","last_fullsized":"2024-09-21T10:27:34.552647+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":96,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.926707+0000","last_change":"2024-09-21T10:25:48.764108+0000","last_active":"2024-09-21T10:27:45.926707+0000","last_peered":"2024-09-21T10:27:45.926707+0000","last_clean":"2024-09-21T10:27:45.926707+0000","last_became_active":"2024-09-21T10:25:48.761361+0000","last_became_peered":"2024-09-21T10:25:48.761361+0000","last_unstale":"2024-09-21T10:27:45.926707+0000","last_undegraded":"2024-09-21T10:27:45.926707+0000","last_fullsized":"2024-09-21T10:27:45.926707+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:49.188529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3116695576,"num_objects":47558,"num_object_clones":0,"num_object_copies":142674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13783,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47558,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":208042,"num_write_kb":8179137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4599,"num_bytes_recovered":301400064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8920772608,"data_stored":8920760392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125642,"ondisk_log_size":125642,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28818,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234457,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2594060,"kb_used_data":1764740,"kb_used_omap":1,"kb_used_meta":829310,"kb_avail":91151092,"statfs":{"total":95995035648,"available":93338718208,"internally_reserved":0,"allocated":1807093760,"data_stored":1805008362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":849213899},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496095,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1288260,"kb_used_data":593468,"kb_used_omap":1,"kb_used_meta":694782,"kb_avail":92456892,"statfs":{"total":95995035648,"available":94675857408,"internally_reserved":0,"allocated":607711232,"data_stored":605634206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":711457223},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403995,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2253088,"kb_used_data":1733844,"kb_used_omap":7,"kb_used_meta":519224,"kb_avail":91492064,"statfs":{"total":95995035648,"available":93687873536,"internally_reserved":0,"allocated":1775456256,"data_stored":1773367837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":531686296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019368,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1233032,"kb_used_data":584192,"kb_used_omap":1,"kb_used_meta":648830,"kb_avail":92512120,"statfs":{"total":95995035648,"available":94732410880,"internally_reserved":0,"allocated":598212608,"data_stored":596131543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":664402374},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435552,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2470968,"kb_used_data":1546544,"kb_used_omap":5,"kb_used_meta":924346,"kb_avail":91274184,"statfs":{"total":95995035648,"available":93464764416,"internally_reserved":0,"allocated":1583661056,"data_stored":1581592119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":946530344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599079,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1348744,"kb_used_data":577084,"kb_used_omap":7,"kb_used_meta":771640,"kb_avail":92396408,"statfs":{"total":95995035648,"available":94613921792,"internally_reserved":0,"allocated":590934016,"data_stored":588861414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":790159821},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762606,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2486292,"kb_used_data":1720196,"kb_used_omap":7,"kb_used_meta":766072,"kb_avail":91258860,"statfs":{"total":95995035648,"available":93449072640,"internally_reserved":0,"allocated":1761480704,"data_stored":1759407212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":784457981},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":389,"seq":1670742278169,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":431012,"kb_used_data":326420,"kb_used_omap":12,"kb_used_meta":104563,"kb_avail":93314140,"statfs":{"total":95995035648,"available":95553679360,"internally_reserved":0,"allocated":334254080,"data_stored":332173414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12595,"internal_metadata":107073229},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11001,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":320339968,"data_stored":320339968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1750011904,"data_stored":1750007832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":584318976,"data_stored":584318976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1564868608,"data_stored":1564868608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":585629696,"data_stored":585629696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1747521536,"data_stored":1747517464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":592642048,"data_stored":592642048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1775439872,"data_stored":1775435800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:27:50.287 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:27:50.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:49 smithi045 ceph-mon[95976]: pgmap v1268: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 13783/142758 objects degraded (9.655%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:27:50.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:49 smithi045 ceph-mon[95976]: from='client.46037 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:27:50.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:49 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2812885798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:27:50.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:50.376 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:27:50.400 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:50.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:50.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:50.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:50.532 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:50.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:50.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:50.633 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:50.664 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:50.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:50.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:50.802 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:50.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:50.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:50.904 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:50.943 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:50.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:51.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:51.088 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:51.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:51.190 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:51.239 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:51.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:51.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:51.340 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:51.378 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:51.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:51.480 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:51.514 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:51.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:51.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:51.650 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:51.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:51.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:51.751 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:51.795 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:51.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:51.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:51.897 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:51.930 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:51.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:51.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:52.032 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:52.065 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:52.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:52.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:52.144 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:51 smithi032 ceph-mon[105748]: pgmap v1269: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 13634/138435 objects degraded (9.849%); 3.7 MiB/s, 59 objects/s recovering 2024-09-21T10:27:52.144 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:51 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 13634/138435 objects degraded (9.849%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:52.144 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:51 smithi032 ceph-mon[101299]: pgmap v1269: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 13634/138435 objects degraded (9.849%); 3.7 MiB/s, 59 objects/s recovering 2024-09-21T10:27:52.144 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:51 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 13634/138435 objects degraded (9.849%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:52.167 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:52.194 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:52.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:52.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:52.198 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:51 smithi045 ceph-mon[95976]: pgmap v1269: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 13634/138435 objects degraded (9.849%); 3.7 MiB/s, 59 objects/s recovering 2024-09-21T10:27:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:51 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 13634/138435 objects degraded (9.849%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:52.295 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:52.326 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:52.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:52.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:52.427 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:52.469 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:52.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:52.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:52.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:52.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:52.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:52.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:52.707 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:52.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:52.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:52.841 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:52.872 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:52.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:52.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:52.957 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:27:52.974 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:53.079 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:53.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:53.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:53.098 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:27:53.098 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:27:53.098 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.098 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45848.0:214060 9.4 9:218f1eb5:::benchmark_data_smithi032_265250_object86258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-21T10:27:53.098 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:53.072769+0000", 2024-09-21T10:27:53.098 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.022172865999999999, 2024-09-21T10:27:53.098 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.022203983, 2024-09-21T10:27:53.098 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45848", 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 214060 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.072769+0000", 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.099 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.072769+0000", 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.072771+0000", 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.567e-06 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.072775+0000", 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5449999999999997e-06 2024-09-21T10:27:53.100 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.072792+0000", 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6977999999999999e-05 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.072800+0000", 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8359999999999992e-06 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.072883+0000", 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3071999999999993e-05 2024-09-21T10:27:53.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.073063+0000", 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017996700000000001 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.073201+0000", 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000137558 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.088305+0000", 2024-09-21T10:27:53.102 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.015104573 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.093272+0000", 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0049664959999999999 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:214059 9.6 e410/378 9:6ee98b4f:::benchmark_data_smithi032_265250_object86257:head v 410'13095, mlcod=410'13092)", 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:53.075616+0000", 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.01932563, 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.019432787, 2024-09-21T10:27:53.103 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.075616+0000", 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.075616+0000", 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:27:53.104 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.075617+0000", 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.8700000000000004e-07 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.075624+0000", 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8519999999999998e-06 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.076127+0000", 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00050251400000000002 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:27:53.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.076136+0000", 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8950000000000004e-06 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.076223+0000", 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7150000000000004e-05 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.076240+0000", 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7312999999999998e-05 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:27:53.106 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:214067 9.e e410/378 9:7e2159bb:::benchmark_data_smithi032_265250_object86265:head v 410'13304, mlcod=410'13302)", 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:53.092444+0000", 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0024983869999999999, 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0026515570000000001, 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092444+0000", 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.107 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092444+0000", 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092445+0000", 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2240000000000001e-06 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092453+0000", 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.96e-06 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.108 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092475+0000", 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2717e-05 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092486+0000", 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.007e-05 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092556+0000", 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0271000000000005e-05 2024-09-21T10:27:53.109 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092582+0000", 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5907000000000001e-05 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:214065 9.f e410/378 9:fdd476d4:::benchmark_data_smithi032_265250_object86263:head v 410'13329, mlcod=410'13327)", 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:53.092522+0000", 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.0024195240000000002, 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0026205880000000001, 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:27:53.110 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092522+0000", 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092522+0000", 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.111 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092523+0000", 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.7e-07 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092528+0000", 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8999999999999997e-06 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092535+0000", 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9600000000000003e-06 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.112 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092540+0000", 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.887e-06 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092630+0000", 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0376000000000003e-05 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.092648+0000", 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7595e-05 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.113 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:214071 9.e e410/378)", 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:53.094428+0000", 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.00051415100000000002, 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00076146600000000003, 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "queued for pg", 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094428+0000", 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.114 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094428+0000", 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094429+0000", 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.173e-06 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094438+0000", 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5560000000000008e-06 2024-09-21T10:27:53.115 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094456+0000", 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8791999999999999e-05 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094465+0000", 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5650000000000002e-06 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.45848.0:214060 9.4 e410/378 ondisk, result = 0)", 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:53.094625+0000", 2024-09-21T10:27:53.116 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.00031646199999999999, 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00060195500000000005, 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094625+0000", 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094625+0000", 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.117 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094626+0000", 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2800000000000005e-07 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094632+0000", 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7420000000000003e-06 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094649+0000", 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6554000000000001e-05 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:27:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094656+0000", 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2740000000000002e-06 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094978+0000", 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000322112 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094987+0000", 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7779999999999996e-06 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.119 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:214068 9.2 e410/378 9:45d897c6:::benchmark_data_smithi032_265250_object86266:head v 410'13485, mlcod=410'13483)", 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:53.094873+0000", 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 6.8620000000000004e-05, 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00041208, 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094873+0000", 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.120 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094873+0000", 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094874+0000", 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.39e-07 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094880+0000", 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7540000000000003e-06 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094891+0000", 2024-09-21T10:27:53.121 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1737e-05 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094898+0000", 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8079999999999999e-06 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094980+0000", 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2233999999999999e-05 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:53.094998+0000", 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8136999999999999e-05 2024-09-21T10:27:53.122 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.123 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:27:53.123 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.123 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:27:53.123 INFO:teuthology.orchestra.run.smithi032.stdout: ], 2024-09-21T10:27:53.123 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 7 2024-09-21T10:27:53.123 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:27:53.180 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:53.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:53.234 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:53.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:53.335 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:53.361 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:53.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:53.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:53.463 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:53.507 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:53.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:53.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:53.609 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:53.648 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:53.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:53.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:53.749 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:53.780 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:53.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:53.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:53.882 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:53.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:53.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:53.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:54.018 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:54.050 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:54.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:54.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:54.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:27:54.152 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:53 smithi032 ceph-mon[101299]: pgmap v1270: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 13634/131667 objects degraded (10.355%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:27:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:53 smithi032 ceph-mon[105748]: pgmap v1270: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 13634/131667 objects degraded (10.355%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:27:54.210 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:54.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:54.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:53 smithi045 ceph-mon[95976]: pgmap v1270: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 13634/131667 objects degraded (10.355%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:27:54.312 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:54.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:54.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:54.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:54.464 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:54.491 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:54.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:54.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:54.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:54.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:54.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:54.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:54.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:54.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:54.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:54.880 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:54.917 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:54.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:55.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:55.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:55.099 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:55.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:55.120 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:55.137 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:55.162 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:55.199 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:55.233 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:55.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:55.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:55.335 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:55.375 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:55.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:55.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:55.476 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:55.508 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:55.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:55.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:55.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:55.775 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:55.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:55.876 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:55.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:55.911 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:55.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:56.012 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:56.038 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:56.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:56.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:56.117 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:55 smithi032 ceph-mon[101299]: pgmap v1271: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 13513/126210 objects degraded (10.707%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:27:56.118 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:55 smithi032 ceph-mon[105748]: pgmap v1271: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 13513/126210 objects degraded (10.707%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:27:56.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:56.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:56.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:56.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:56.276 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:55 smithi045 ceph-mon[95976]: pgmap v1271: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 13513/126210 objects degraded (10.707%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:27:56.326 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:56.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:56.428 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:56.465 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:56.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:56.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:56.567 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:56.605 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:56.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:56.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:56.707 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:56.742 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:56.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:56.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:56.844 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:56.886 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:56.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:56.971 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:27:56.987 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:57.014 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:57.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:57.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:57.115 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:57.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:57.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:57.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:57.263 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:27:57.268 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/744058771' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:27:57.268 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:56 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 13513/126210 objects degraded (10.707%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:57.271 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/744058771' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:27:57.271 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:56 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 13513/126210 objects degraded (10.707%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/744058771' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:27:57.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:56 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 13513/126210 objects degraded (10.707%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:27:57.294 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:57.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:57.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:27:57.396 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:57.424 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:57.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:57.526 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:27:57.559 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:57.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:57.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:27:57.660 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:57.695 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:57.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:57.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:57.841 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:57.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:57.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:57.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:57.986 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:57.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:58.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:58.136 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:58.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:58.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:58.235 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:57 smithi032 ceph-mon[101299]: pgmap v1272: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 13513/120063 objects degraded (11.255%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:27:58.235 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:57 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:58.235 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:57 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/744058771' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:27:58.235 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:57 smithi032 ceph-mon[101299]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T10:27:58.235 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:57 smithi032 ceph-mon[105748]: pgmap v1272: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 13513/120063 objects degraded (11.255%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:27:58.235 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:57 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:58.235 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:57 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/744058771' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:27:58.236 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:57 smithi032 ceph-mon[105748]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T10:27:58.238 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:58.276 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:58.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:58.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:57 smithi045 ceph-mon[95976]: pgmap v1272: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 13513/120063 objects degraded (11.255%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:27:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:57 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:27:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:57 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/744058771' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:27:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:57 smithi045 ceph-mon[95976]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T10:27:58.378 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:58.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:58.429 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:58.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:58.530 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:58.560 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:58.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:58.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:58.662 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:58.677 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:27:58.678 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:27:58.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:58.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:27:58.801 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:58.841 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:58.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:58.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:27:58.991 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:58.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:58.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:27:59.093 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:59.122 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:59.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:59.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:59.224 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:59.263 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:59.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:59.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:59.364 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:27:59.402 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:59.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:59.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:27:59.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:27:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:59.551 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:59.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:27:59.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:27:59.683 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:59.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:59.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:27:59.784 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:27:59.818 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:27:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:59.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:27:59.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:27:59.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:27:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:27:59] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-21T10:27:59.959 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:27:59.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:27:59.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:00.060 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:00.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:00.100 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:00.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:00.201 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:00.234 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:00.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:00.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:59 smithi045 ceph-mon[95976]: pgmap v1274: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 13513/119247 objects degraded (11.332%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:28:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:27:59 smithi045 ceph-mon[95976]: from='client.55681 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:28:00.336 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:00.375 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:00.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:00.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:59 smithi032 ceph-mon[101299]: pgmap v1274: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 13513/119247 objects degraded (11.332%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:28:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:27:59 smithi032 ceph-mon[101299]: from='client.55681 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:28:00.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:59 smithi032 ceph-mon[105748]: pgmap v1274: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 13513/119247 objects degraded (11.332%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:28:00.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:27:59 smithi032 ceph-mon[105748]: from='client.55681 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:28:00.477 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:00.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:00.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:00.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:00.612 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:00.644 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:00.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:00.746 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:00.788 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:00.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:00.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:00.890 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:00.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:00.930 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:00.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:00.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:28:01.031 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:01.060 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:01.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:01.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:01.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:01.189 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:01.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:01.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:01.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:01.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:01.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:01.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:01.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:01.491 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:01.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:01.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:01.642 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:01.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:01.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:01.744 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:01.768 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:01.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:01.870 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:01.903 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:01.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:01.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:02.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:02.044 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:02.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:02.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:02.183 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:02.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:02.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:02.284 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:01 smithi045 ceph-mon[95976]: pgmap v1275: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 13382/113874 objects degraded (11.752%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:01 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 13382/113874 objects degraded (11.752%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:02.320 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:02.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:02.421 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:02.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:02.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:02.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:01 smithi032 ceph-mon[101299]: pgmap v1275: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 13382/113874 objects degraded (11.752%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:02.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:01 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 13382/113874 objects degraded (11.752%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:01 smithi032 ceph-mon[105748]: pgmap v1275: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 13382/113874 objects degraded (11.752%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:01 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 13382/113874 objects degraded (11.752%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:02.557 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:02.596 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:02.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:02.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:02.698 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:02.738 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:02.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:02.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:02.840 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:02.883 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:02.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:02.985 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:03.018 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:03.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:03.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:03.119 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:03.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:03.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:03.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:03.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:03.302 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:03.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:03.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:03.439 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:03.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:03.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:03.587 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:03.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:03.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:03.688 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:03.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:03.730 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:03.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:03.771 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1274,"stamp":"2024-09-21T10:27:58.646927+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2607880189,"num_objects":39749,"num_object_clones":0,"num_object_copies":119247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13513,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39749,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":216787,"num_write_kb":8191762,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5154,"num_bytes_recovered":335877765,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128253,"ondisk_log_size":128253,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12711192,"kb_used_data":7431448,"kb_used_omap":45,"kb_used_meta":5279378,"kb_avail":737250024,"statfs":{"total":767960285184,"available":754944024576,"internally_reserved":0,"allocated":7609802752,"data_stored":7593112711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46417,"internal_metadata":5406083759},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,2,5,10,12,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-513605632,"num_objects":-7837,"num_object_clones":0,"num_object_copies":-23511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7837,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":270,"num_bytes_recovered":17694720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005410"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":699,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.952713+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:27:45.952713+0000","last_peered":"2024-09-21T10:27:45.952713+0000","last_clean":"2024-09-21T10:27:45.952713+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:27:45.952713+0000","last_undegraded":"2024-09-21T10:27:45.952713+0000","last_fullsized":"2024-09-21T10:27:45.952713+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":415,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.908644+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:27:44.908644+0000","last_peered":"2024-09-21T10:27:44.908644+0000","last_clean":"2024-09-21T10:27:44.908644+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:27:44.908644+0000","last_undegraded":"2024-09-21T10:27:44.908644+0000","last_fullsized":"2024-09-21T10:27:44.908644+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":111,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.905111+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:27:44.905111+0000","last_peered":"2024-09-21T10:27:44.905111+0000","last_clean":"2024-09-21T10:27:44.905111+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:27:44.905111+0000","last_undegraded":"2024-09-21T10:27:44.905111+0000","last_fullsized":"2024-09-21T10:27:44.905111+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":67,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.981008+0000","last_change":"2024-09-21T10:25:48.765404+0000","last_active":"2024-09-21T10:27:56.981008+0000","last_peered":"2024-09-21T10:27:56.981008+0000","last_clean":"2024-09-21T10:27:56.981008+0000","last_became_active":"2024-09-21T10:25:48.764230+0000","last_became_peered":"2024-09-21T10:25:48.764230+0000","last_unstale":"2024-09-21T10:27:56.981008+0000","last_undegraded":"2024-09-21T10:27:56.981008+0000","last_fullsized":"2024-09-21T10:27:56.981008+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:38.551941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":68,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.981325+0000","last_change":"2024-09-21T10:25:48.764391+0000","last_active":"2024-09-21T10:27:56.981325+0000","last_peered":"2024-09-21T10:27:56.981325+0000","last_clean":"2024-09-21T10:27:56.981325+0000","last_became_active":"2024-09-21T10:25:48.763375+0000","last_became_peered":"2024-09-21T10:25:48.763375+0000","last_unstale":"2024-09-21T10:27:56.981325+0000","last_undegraded":"2024-09-21T10:27:56.981325+0000","last_fullsized":"2024-09-21T10:27:56.981325+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:03.457716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":386,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.908369+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:27:44.908369+0000","last_peered":"2024-09-21T10:27:44.908369+0000","last_clean":"2024-09-21T10:27:44.908369+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:27:44.908369+0000","last_undegraded":"2024-09-21T10:27:44.908369+0000","last_fullsized":"2024-09-21T10:27:44.908369+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":719,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.946821+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:27:45.946821+0000","last_peered":"2024-09-21T10:27:45.946821+0000","last_clean":"2024-09-21T10:27:45.946821+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:27:45.946821+0000","last_undegraded":"2024-09-21T10:27:45.946821+0000","last_fullsized":"2024-09-21T10:27:45.946821+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":482,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904575+0000","last_change":"2024-09-21T10:25:48.752712+0000","last_active":"2024-09-21T10:27:44.904575+0000","last_peered":"2024-09-21T10:27:44.904575+0000","last_clean":"2024-09-21T10:27:44.904575+0000","last_became_active":"2024-09-21T10:25:48.752204+0000","last_became_peered":"2024-09-21T10:25:48.752204+0000","last_unstale":"2024-09-21T10:27:44.904575+0000","last_undegraded":"2024-09-21T10:27:44.904575+0000","last_fullsized":"2024-09-21T10:27:44.904575+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:40.068520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"411'13624","reported_seq":22850,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:57.050035+0000","last_change":"2024-09-21T10:27:44.942565+0000","last_active":"2024-09-21T10:27:57.050035+0000","last_peered":"2024-09-21T10:27:57.050035+0000","last_clean":"2024-09-21T10:27:57.050035+0000","last_became_active":"2024-09-21T10:27:44.909231+0000","last_became_peered":"2024-09-21T10:27:44.909231+0000","last_unstale":"2024-09-21T10:27:57.050035+0000","last_undegraded":"2024-09-21T10:27:57.050035+0000","last_fullsized":"2024-09-21T10:27:57.050035+0000","mapping_epoch":408,"log_start":"385'4500","ondisk_log_start":"385'4500","created":378,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":9124,"log_dups_size":0,"ondisk_log_size":9124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:13.740223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156762112,"num_objects":2392,"num_object_clones":0,"num_object_copies":7176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13624,"num_write_kb":512512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4576,"num_bytes_recovered":299892736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":116,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.909051+0000","last_change":"2024-09-21T10:25:48.755730+0000","last_active":"2024-09-21T10:27:44.909051+0000","last_peered":"2024-09-21T10:27:44.909051+0000","last_clean":"2024-09-21T10:27:44.909051+0000","last_became_active":"2024-09-21T10:25:48.755344+0000","last_became_peered":"2024-09-21T10:25:48.755344+0000","last_unstale":"2024-09-21T10:27:44.909051+0000","last_undegraded":"2024-09-21T10:27:44.909051+0000","last_fullsized":"2024-09-21T10:27:44.909051+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:37.725090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":119,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904408+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:27:44.904408+0000","last_peered":"2024-09-21T10:27:44.904408+0000","last_clean":"2024-09-21T10:27:44.904408+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:27:44.904408+0000","last_undegraded":"2024-09-21T10:27:44.904408+0000","last_fullsized":"2024-09-21T10:27:44.904408+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"410'13481","reported_seq":13557,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:55.789767+0000","last_change":"2024-09-21T10:24:25.829780+0000","last_active":"2024-09-21T10:27:55.789767+0000","last_peered":"2024-09-21T10:27:55.789767+0000","last_clean":"2024-09-21T10:27:55.789767+0000","last_became_active":"2024-09-21T10:24:25.829509+0000","last_became_peered":"2024-09-21T10:24:25.829509+0000","last_unstale":"2024-09-21T10:27:55.789767+0000","last_undegraded":"2024-09-21T10:27:55.789767+0000","last_fullsized":"2024-09-21T10:27:55.789767+0000","mapping_epoch":378,"log_start":"386'6100","ondisk_log_start":"386'6100","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7381,"log_dups_size":0,"ondisk_log_size":7381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:30.108791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161677312,"num_objects":2467,"num_object_clones":0,"num_object_copies":7401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13481,"num_write_kb":510336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":88,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.908822+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:27:44.908822+0000","last_peered":"2024-09-21T10:27:44.908822+0000","last_clean":"2024-09-21T10:27:44.908822+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:27:44.908822+0000","last_undegraded":"2024-09-21T10:27:44.908822+0000","last_fullsized":"2024-09-21T10:27:44.908822+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.987445+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:27:44.987445+0000","last_peered":"2024-09-21T10:27:44.987445+0000","last_clean":"2024-09-21T10:27:44.987445+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:27:44.987445+0000","last_undegraded":"2024-09-21T10:27:44.987445+0000","last_fullsized":"2024-09-21T10:27:44.987445+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"410'13568","reported_seq":13609,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:55.843070+0000","last_change":"2024-09-21T10:24:25.825494+0000","last_active":"2024-09-21T10:27:55.843070+0000","last_peered":"2024-09-21T10:27:55.843070+0000","last_clean":"2024-09-21T10:27:55.843070+0000","last_became_active":"2024-09-21T10:24:25.825196+0000","last_became_peered":"2024-09-21T10:24:25.825196+0000","last_unstale":"2024-09-21T10:27:55.843070+0000","last_undegraded":"2024-09-21T10:27:55.843070+0000","last_fullsized":"2024-09-21T10:27:55.843070+0000","mapping_epoch":378,"log_start":"385'4700","ondisk_log_start":"385'4700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8868,"log_dups_size":0,"ondisk_log_size":8868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:50.985537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160956416,"num_objects":2456,"num_object_clones":0,"num_object_copies":7368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13568,"num_write_kb":512768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":410,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.908893+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:27:44.908893+0000","last_peered":"2024-09-21T10:27:44.908893+0000","last_clean":"2024-09-21T10:27:44.908893+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:27:44.908893+0000","last_undegraded":"2024-09-21T10:27:44.908893+0000","last_fullsized":"2024-09-21T10:27:44.908893+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":430,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904566+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:27:44.904566+0000","last_peered":"2024-09-21T10:27:44.904566+0000","last_clean":"2024-09-21T10:27:44.904566+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:27:44.904566+0000","last_undegraded":"2024-09-21T10:27:44.904566+0000","last_fullsized":"2024-09-21T10:27:44.904566+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"410'13240","reported_seq":13282,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:55.901194+0000","last_change":"2024-09-21T10:24:25.846770+0000","last_active":"2024-09-21T10:27:55.901194+0000","last_peered":"2024-09-21T10:27:55.901194+0000","last_clean":"2024-09-21T10:27:55.901194+0000","last_became_active":"2024-09-21T10:24:25.846530+0000","last_became_peered":"2024-09-21T10:24:25.846530+0000","last_unstale":"2024-09-21T10:27:55.901194+0000","last_undegraded":"2024-09-21T10:27:55.901194+0000","last_fullsized":"2024-09-21T10:27:55.901194+0000","mapping_epoch":378,"log_start":"385'4400","ondisk_log_start":"385'4400","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8840,"log_dups_size":0,"ondisk_log_size":8840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:50.827496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153878528,"num_objects":2348,"num_object_clones":0,"num_object_copies":7044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13240,"num_write_kb":498816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":344,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904786+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:27:44.904786+0000","last_peered":"2024-09-21T10:27:44.904786+0000","last_clean":"2024-09-21T10:27:44.904786+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:27:44.904786+0000","last_undegraded":"2024-09-21T10:27:44.904786+0000","last_fullsized":"2024-09-21T10:27:44.904786+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":258,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.982941+0000","last_change":"2024-09-21T10:25:48.763096+0000","last_active":"2024-09-21T10:27:56.982941+0000","last_peered":"2024-09-21T10:27:56.982941+0000","last_clean":"2024-09-21T10:27:56.982941+0000","last_became_active":"2024-09-21T10:25:48.761327+0000","last_became_peered":"2024-09-21T10:25:48.761327+0000","last_unstale":"2024-09-21T10:27:56.982941+0000","last_undegraded":"2024-09-21T10:27:56.982941+0000","last_fullsized":"2024-09-21T10:27:56.982941+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:23.154333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"410'13553","reported_seq":13629,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:54.416891+0000","last_change":"2024-09-21T10:24:25.846977+0000","last_active":"2024-09-21T10:27:54.416891+0000","last_peered":"2024-09-21T10:27:54.416891+0000","last_clean":"2024-09-21T10:27:54.416891+0000","last_became_active":"2024-09-21T10:24:25.846670+0000","last_became_peered":"2024-09-21T10:24:25.846670+0000","last_unstale":"2024-09-21T10:27:54.416891+0000","last_undegraded":"2024-09-21T10:27:54.416891+0000","last_fullsized":"2024-09-21T10:27:54.416891+0000","mapping_epoch":378,"log_start":"386'6700","ondisk_log_start":"386'6700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6853,"log_dups_size":0,"ondisk_log_size":6853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:28.387249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166395904,"num_objects":2539,"num_object_clones":0,"num_object_copies":7617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13553,"num_write_kb":514944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":695,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.908287+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:27:44.908287+0000","last_peered":"2024-09-21T10:27:44.908287+0000","last_clean":"2024-09-21T10:27:44.908287+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:27:44.908287+0000","last_undegraded":"2024-09-21T10:27:44.908287+0000","last_fullsized":"2024-09-21T10:27:44.908287+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":760,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.945762+0000","last_change":"2024-09-21T10:25:48.755999+0000","last_active":"2024-09-21T10:27:45.945762+0000","last_peered":"2024-09-21T10:27:45.945762+0000","last_clean":"2024-09-21T10:27:45.945762+0000","last_became_active":"2024-09-21T10:25:48.754257+0000","last_became_peered":"2024-09-21T10:25:48.754257+0000","last_unstale":"2024-09-21T10:27:45.945762+0000","last_undegraded":"2024-09-21T10:27:45.945762+0000","last_fullsized":"2024-09-21T10:27:45.945762+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:06.025116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":107,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.990494+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:27:44.990494+0000","last_peered":"2024-09-21T10:27:44.990494+0000","last_clean":"2024-09-21T10:27:44.990494+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:27:44.990494+0000","last_undegraded":"2024-09-21T10:27:44.990494+0000","last_fullsized":"2024-09-21T10:27:44.990494+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"410'13668","reported_seq":13744,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:55.836008+0000","last_change":"2024-09-21T10:24:25.847167+0000","last_active":"2024-09-21T10:27:55.836008+0000","last_peered":"2024-09-21T10:27:55.836008+0000","last_clean":"2024-09-21T10:27:55.836008+0000","last_became_active":"2024-09-21T10:24:25.846878+0000","last_became_peered":"2024-09-21T10:24:25.846878+0000","last_unstale":"2024-09-21T10:27:55.836008+0000","last_undegraded":"2024-09-21T10:27:55.836008+0000","last_fullsized":"2024-09-21T10:27:55.836008+0000","mapping_epoch":378,"log_start":"386'6300","ondisk_log_start":"386'6300","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7368,"log_dups_size":0,"ondisk_log_size":7368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:32.840244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163971072,"num_objects":2502,"num_object_clones":0,"num_object_copies":7506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13668,"num_write_kb":517440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":468,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904338+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:27:44.904338+0000","last_peered":"2024-09-21T10:27:44.904338+0000","last_clean":"2024-09-21T10:27:44.904338+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:27:44.904338+0000","last_undegraded":"2024-09-21T10:27:44.904338+0000","last_fullsized":"2024-09-21T10:27:44.904338+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":420,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904705+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:27:44.904705+0000","last_peered":"2024-09-21T10:27:44.904705+0000","last_clean":"2024-09-21T10:27:44.904705+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:27:44.904705+0000","last_undegraded":"2024-09-21T10:27:44.904705+0000","last_fullsized":"2024-09-21T10:27:44.904705+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.988395+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:27:44.988395+0000","last_peered":"2024-09-21T10:27:44.988395+0000","last_clean":"2024-09-21T10:27:44.988395+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:27:44.988395+0000","last_undegraded":"2024-09-21T10:27:44.988395+0000","last_fullsized":"2024-09-21T10:27:44.988395+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"410'13526","reported_seq":13568,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:54.178477+0000","last_change":"2024-09-21T10:24:25.828673+0000","last_active":"2024-09-21T10:27:54.178477+0000","last_peered":"2024-09-21T10:27:54.178477+0000","last_clean":"2024-09-21T10:27:54.178477+0000","last_became_active":"2024-09-21T10:24:25.828433+0000","last_became_peered":"2024-09-21T10:24:25.828433+0000","last_unstale":"2024-09-21T10:27:54.178477+0000","last_undegraded":"2024-09-21T10:27:54.178477+0000","last_fullsized":"2024-09-21T10:27:54.178477+0000","mapping_epoch":378,"log_start":"386'6500","ondisk_log_start":"386'6500","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:07.654019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163446784,"num_objects":2494,"num_object_clones":0,"num_object_copies":7482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13526,"num_write_kb":512640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":107,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.987304+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:27:44.987304+0000","last_peered":"2024-09-21T10:27:44.987304+0000","last_clean":"2024-09-21T10:27:44.987304+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:27:44.987304+0000","last_undegraded":"2024-09-21T10:27:44.987304+0000","last_fullsized":"2024-09-21T10:27:44.987304+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":107,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.988722+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:27:44.988722+0000","last_peered":"2024-09-21T10:27:44.988722+0000","last_clean":"2024-09-21T10:27:44.988722+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:27:44.988722+0000","last_undegraded":"2024-09-21T10:27:44.988722+0000","last_fullsized":"2024-09-21T10:27:44.988722+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"410'13560","reported_seq":13636,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:54.379452+0000","last_change":"2024-09-21T10:24:25.847020+0000","last_active":"2024-09-21T10:27:54.379452+0000","last_peered":"2024-09-21T10:27:54.379452+0000","last_clean":"2024-09-21T10:27:54.379452+0000","last_became_active":"2024-09-21T10:24:25.846622+0000","last_became_peered":"2024-09-21T10:24:25.846622+0000","last_unstale":"2024-09-21T10:27:54.379452+0000","last_undegraded":"2024-09-21T10:27:54.379452+0000","last_fullsized":"2024-09-21T10:27:54.379452+0000","mapping_epoch":378,"log_start":"386'6700","ondisk_log_start":"386'6700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6860,"log_dups_size":0,"ondisk_log_size":6860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:54:04.668262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164888576,"num_objects":2516,"num_object_clones":0,"num_object_copies":7548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13560,"num_write_kb":514432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":646,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.905178+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:27:44.905178+0000","last_peered":"2024-09-21T10:27:44.905178+0000","last_clean":"2024-09-21T10:27:44.905178+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:27:44.905178+0000","last_undegraded":"2024-09-21T10:27:44.905178+0000","last_fullsized":"2024-09-21T10:27:44.905178+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":728,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.970589+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:27:45.970589+0000","last_peered":"2024-09-21T10:27:45.970589+0000","last_clean":"2024-09-21T10:27:45.970589+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:27:45.970589+0000","last_undegraded":"2024-09-21T10:27:45.970589+0000","last_fullsized":"2024-09-21T10:27:45.970589+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":420,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.940123+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:27:45.940123+0000","last_peered":"2024-09-21T10:27:45.940123+0000","last_clean":"2024-09-21T10:27:45.940123+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:27:45.940123+0000","last_undegraded":"2024-09-21T10:27:45.940123+0000","last_fullsized":"2024-09-21T10:27:45.940123+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"410'13370","reported_seq":14041,"reported_epoch":410,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:27:54.431086+0000","last_change":"2024-09-21T10:27:43.773504+0000","last_active":"2024-09-21T10:27:54.431086+0000","last_peered":"2024-09-21T10:27:54.431086+0000","last_clean":"2024-09-21T10:24:52.081220+0000","last_became_active":"2024-09-21T10:25:50.482809+0000","last_became_peered":"2024-09-21T10:25:50.482809+0000","last_unstale":"2024-09-21T10:27:54.431086+0000","last_undegraded":"2024-09-21T10:25:49.768122+0000","last_fullsized":"2024-09-21T10:25:49.744590+0000","mapping_epoch":390,"log_start":"386'6500","ondisk_log_start":"386'6500","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6870,"log_dups_size":0,"ondisk_log_size":6870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165019648,"num_objects":2518,"num_object_clones":0,"num_object_copies":7554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4244,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13370,"num_write_kb":508416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":292,"num_bytes_recovered":19136512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-1726}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":695,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.908161+0000","last_change":"2024-09-21T10:25:48.754725+0000","last_active":"2024-09-21T10:27:44.908161+0000","last_peered":"2024-09-21T10:27:44.908161+0000","last_clean":"2024-09-21T10:27:44.908161+0000","last_became_active":"2024-09-21T10:25:48.754341+0000","last_became_peered":"2024-09-21T10:25:48.754341+0000","last_unstale":"2024-09-21T10:27:44.908161+0000","last_undegraded":"2024-09-21T10:27:44.908161+0000","last_fullsized":"2024-09-21T10:27:44.908161+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:25.191555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:25.191555+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:08.981054+0000","scrub_duration":0.0015044590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":723,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.906029+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:27:44.906029+0000","last_peered":"2024-09-21T10:27:44.906029+0000","last_clean":"2024-09-21T10:27:44.906029+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:27:44.906029+0000","last_undegraded":"2024-09-21T10:27:44.906029+0000","last_fullsized":"2024-09-21T10:27:44.906029+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":659,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.940999+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:27:45.940999+0000","last_peered":"2024-09-21T10:27:45.940999+0000","last_clean":"2024-09-21T10:27:45.940999+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:27:45.940999+0000","last_undegraded":"2024-09-21T10:27:45.940999+0000","last_fullsized":"2024-09-21T10:27:45.940999+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":386,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904937+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:27:44.904937+0000","last_peered":"2024-09-21T10:27:44.904937+0000","last_clean":"2024-09-21T10:27:44.904937+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:27:44.904937+0000","last_undegraded":"2024-09-21T10:27:44.904937+0000","last_fullsized":"2024-09-21T10:27:44.904937+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"410'13339","reported_seq":13393,"reported_epoch":410,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:27:55.910845+0000","last_change":"2024-09-21T10:25:50.516299+0000","last_active":"2024-09-21T10:27:55.910845+0000","last_peered":"2024-09-21T10:27:55.910845+0000","last_clean":"2024-09-21T10:24:52.080442+0000","last_became_active":"2024-09-21T10:25:50.497794+0000","last_became_peered":"2024-09-21T10:25:50.497794+0000","last_unstale":"2024-09-21T10:27:55.910845+0000","last_undegraded":"2024-09-21T10:25:49.773994+0000","last_fullsized":"2024-09-21T10:25:49.744196+0000","mapping_epoch":390,"log_start":"385'4500","ondisk_log_start":"385'4500","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8839,"log_dups_size":0,"ondisk_log_size":8839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:02.445442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156696576,"num_objects":2391,"num_object_clones":0,"num_object_copies":7173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4583,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13339,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":-2192}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":676,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.936224+0000","last_change":"2024-09-21T10:24:27.359111+0000","last_active":"2024-09-21T10:27:45.936224+0000","last_peered":"2024-09-21T10:27:45.936224+0000","last_clean":"2024-09-21T10:27:45.936224+0000","last_became_active":"2024-09-21T10:24:24.825850+0000","last_became_peered":"2024-09-21T10:24:24.825850+0000","last_unstale":"2024-09-21T10:27:45.936224+0000","last_undegraded":"2024-09-21T10:27:45.936224+0000","last_fullsized":"2024-09-21T10:27:45.936224+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:27.359065+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:24:27.359065+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:06.871319+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1067,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.908195+0000","last_change":"2024-09-21T10:25:48.755667+0000","last_active":"2024-09-21T10:27:44.908195+0000","last_peered":"2024-09-21T10:27:44.908195+0000","last_clean":"2024-09-21T10:27:44.908195+0000","last_became_active":"2024-09-21T10:25:48.755060+0000","last_became_peered":"2024-09-21T10:25:48.755060+0000","last_unstale":"2024-09-21T10:27:44.908195+0000","last_undegraded":"2024-09-21T10:27:44.908195+0000","last_fullsized":"2024-09-21T10:27:44.908195+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:09.230030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":261,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.981820+0000","last_change":"2024-09-21T10:25:48.754659+0000","last_active":"2024-09-21T10:27:56.981820+0000","last_peered":"2024-09-21T10:27:56.981820+0000","last_clean":"2024-09-21T10:27:56.981820+0000","last_became_active":"2024-09-21T10:25:48.754230+0000","last_became_peered":"2024-09-21T10:25:48.754230+0000","last_unstale":"2024-09-21T10:27:56.981820+0000","last_undegraded":"2024-09-21T10:27:56.981820+0000","last_fullsized":"2024-09-21T10:27:56.981820+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:12.555868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.981122+0000","last_change":"2024-09-21T10:25:48.761249+0000","last_active":"2024-09-21T10:27:56.981122+0000","last_peered":"2024-09-21T10:27:56.981122+0000","last_clean":"2024-09-21T10:27:56.981122+0000","last_became_active":"2024-09-21T10:25:48.759359+0000","last_became_peered":"2024-09-21T10:25:48.759359+0000","last_unstale":"2024-09-21T10:27:56.981122+0000","last_undegraded":"2024-09-21T10:27:56.981122+0000","last_fullsized":"2024-09-21T10:27:56.981122+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:33.510936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"410'13518","reported_seq":13560,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:55.894932+0000","last_change":"2024-09-21T10:24:25.827798+0000","last_active":"2024-09-21T10:27:55.894932+0000","last_peered":"2024-09-21T10:27:55.894932+0000","last_clean":"2024-09-21T10:27:55.894932+0000","last_became_active":"2024-09-21T10:24:25.827230+0000","last_became_peered":"2024-09-21T10:24:25.827230+0000","last_unstale":"2024-09-21T10:27:55.894932+0000","last_undegraded":"2024-09-21T10:27:55.894932+0000","last_fullsized":"2024-09-21T10:27:55.894932+0000","mapping_epoch":378,"log_start":"385'4600","ondisk_log_start":"385'4600","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8918,"log_dups_size":0,"ondisk_log_size":8918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:28:46.430403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166068224,"num_objects":2534,"num_object_clones":0,"num_object_copies":7602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13518,"num_write_kb":513664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":711,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.923867+0000","last_change":"2024-09-21T10:25:48.755509+0000","last_active":"2024-09-21T10:27:45.923867+0000","last_peered":"2024-09-21T10:27:45.923867+0000","last_clean":"2024-09-21T10:27:45.923867+0000","last_became_active":"2024-09-21T10:25:48.754558+0000","last_became_peered":"2024-09-21T10:25:48.754558+0000","last_unstale":"2024-09-21T10:27:45.923867+0000","last_undegraded":"2024-09-21T10:27:45.923867+0000","last_fullsized":"2024-09-21T10:27:45.923867+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:11.262503+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1259,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.944101+0000","last_change":"2024-09-21T10:25:48.753359+0000","last_active":"2024-09-21T10:27:45.944101+0000","last_peered":"2024-09-21T10:27:45.944101+0000","last_clean":"2024-09-21T10:27:45.944101+0000","last_became_active":"2024-09-21T10:25:48.752653+0000","last_became_peered":"2024-09-21T10:25:48.752653+0000","last_unstale":"2024-09-21T10:27:45.944101+0000","last_undegraded":"2024-09-21T10:27:45.944101+0000","last_fullsized":"2024-09-21T10:27:45.944101+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:55.095131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":407,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.936087+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:27:45.936087+0000","last_peered":"2024-09-21T10:27:45.936087+0000","last_clean":"2024-09-21T10:27:45.936087+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:27:45.936087+0000","last_undegraded":"2024-09-21T10:27:45.936087+0000","last_fullsized":"2024-09-21T10:27:45.936087+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":447,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.940215+0000","last_change":"2024-09-21T10:25:48.766755+0000","last_active":"2024-09-21T10:27:45.940215+0000","last_peered":"2024-09-21T10:27:45.940215+0000","last_clean":"2024-09-21T10:27:45.940215+0000","last_became_active":"2024-09-21T10:25:48.766298+0000","last_became_peered":"2024-09-21T10:25:48.766298+0000","last_unstale":"2024-09-21T10:27:45.940215+0000","last_undegraded":"2024-09-21T10:27:45.940215+0000","last_fullsized":"2024-09-21T10:27:45.940215+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:18.788447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"410'13657","reported_seq":13716,"reported_epoch":410,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:27:55.898669+0000","last_change":"2024-09-21T10:25:50.515832+0000","last_active":"2024-09-21T10:27:55.898669+0000","last_peered":"2024-09-21T10:27:55.898669+0000","last_clean":"2024-09-21T10:24:52.091731+0000","last_became_active":"2024-09-21T10:25:50.504214+0000","last_became_peered":"2024-09-21T10:25:50.504214+0000","last_unstale":"2024-09-21T10:27:55.898669+0000","last_undegraded":"2024-09-21T10:25:49.774199+0000","last_fullsized":"2024-09-21T10:25:49.743948+0000","mapping_epoch":390,"log_start":"385'4800","ondisk_log_start":"385'4800","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8857,"log_dups_size":0,"ondisk_log_size":8857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:49.426271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162594816,"num_objects":2481,"num_object_clones":0,"num_object_copies":7443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13657,"num_write_kb":516416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":-2205}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":661,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904947+0000","last_change":"2024-09-21T10:25:48.757672+0000","last_active":"2024-09-21T10:27:44.904947+0000","last_peered":"2024-09-21T10:27:44.904947+0000","last_clean":"2024-09-21T10:27:44.904947+0000","last_became_active":"2024-09-21T10:25:48.757296+0000","last_became_peered":"2024-09-21T10:25:48.757296+0000","last_unstale":"2024-09-21T10:27:44.904947+0000","last_undegraded":"2024-09-21T10:27:44.904947+0000","last_fullsized":"2024-09-21T10:27:44.904947+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:31.244809+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:31.244809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:32.654430+0000","scrub_duration":0.0015828649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1154,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.905461+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:27:44.905461+0000","last_peered":"2024-09-21T10:27:44.905461+0000","last_clean":"2024-09-21T10:27:44.905461+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:27:44.905461+0000","last_undegraded":"2024-09-21T10:27:44.905461+0000","last_fullsized":"2024-09-21T10:27:44.905461+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":122,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.988444+0000","last_change":"2024-09-21T10:25:48.753400+0000","last_active":"2024-09-21T10:27:44.988444+0000","last_peered":"2024-09-21T10:27:44.988444+0000","last_clean":"2024-09-21T10:27:44.988444+0000","last_became_active":"2024-09-21T10:25:48.753088+0000","last_became_peered":"2024-09-21T10:25:48.753088+0000","last_unstale":"2024-09-21T10:27:44.988444+0000","last_undegraded":"2024-09-21T10:27:44.988444+0000","last_fullsized":"2024-09-21T10:27:44.988444+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:49.107566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":407,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.936566+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:27:45.936566+0000","last_peered":"2024-09-21T10:27:45.936566+0000","last_clean":"2024-09-21T10:27:45.936566+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:27:45.936566+0000","last_undegraded":"2024-09-21T10:27:45.936566+0000","last_fullsized":"2024-09-21T10:27:45.936566+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"410'13476","reported_seq":13518,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:55.895872+0000","last_change":"2024-09-21T10:24:25.831730+0000","last_active":"2024-09-21T10:27:55.895872+0000","last_peered":"2024-09-21T10:27:55.895872+0000","last_clean":"2024-09-21T10:27:55.895872+0000","last_became_active":"2024-09-21T10:24:25.831416+0000","last_became_peered":"2024-09-21T10:24:25.831416+0000","last_unstale":"2024-09-21T10:27:55.895872+0000","last_undegraded":"2024-09-21T10:27:55.895872+0000","last_fullsized":"2024-09-21T10:27:55.895872+0000","mapping_epoch":378,"log_start":"385'4600","ondisk_log_start":"385'4600","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8876,"log_dups_size":0,"ondisk_log_size":8876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.763992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164888576,"num_objects":2516,"num_object_clones":0,"num_object_copies":7548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13476,"num_write_kb":511744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":494,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.905176+0000","last_change":"2024-09-21T10:24:24.828950+0000","last_active":"2024-09-21T10:27:44.905176+0000","last_peered":"2024-09-21T10:27:44.905176+0000","last_clean":"2024-09-21T10:27:44.905176+0000","last_became_active":"2024-09-21T10:24:24.828550+0000","last_became_peered":"2024-09-21T10:24:24.828550+0000","last_unstale":"2024-09-21T10:27:44.905176+0000","last_undegraded":"2024-09-21T10:27:44.905176+0000","last_fullsized":"2024-09-21T10:27:44.905176+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:29.313198+0000","scrub_duration":0.001370488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":359,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904811+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:27:44.904811+0000","last_peered":"2024-09-21T10:27:44.904811+0000","last_clean":"2024-09-21T10:27:44.904811+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:27:44.904811+0000","last_undegraded":"2024-09-21T10:27:44.904811+0000","last_fullsized":"2024-09-21T10:27:44.904811+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":1027,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.936798+0000","last_change":"2024-09-21T10:25:48.760586+0000","last_active":"2024-09-21T10:27:45.936798+0000","last_peered":"2024-09-21T10:27:45.936798+0000","last_clean":"2024-09-21T10:27:45.936798+0000","last_became_active":"2024-09-21T10:25:48.760042+0000","last_became_peered":"2024-09-21T10:25:48.760042+0000","last_unstale":"2024-09-21T10:27:45.936798+0000","last_undegraded":"2024-09-21T10:27:45.936798+0000","last_fullsized":"2024-09-21T10:27:45.936798+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:41.526062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":443,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.938905+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:27:45.938905+0000","last_peered":"2024-09-21T10:27:45.938905+0000","last_clean":"2024-09-21T10:27:45.938905+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:27:45.938905+0000","last_undegraded":"2024-09-21T10:27:45.938905+0000","last_fullsized":"2024-09-21T10:27:45.938905+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"410'13351","reported_seq":13393,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:54.125820+0000","last_change":"2024-09-21T10:24:25.831693+0000","last_active":"2024-09-21T10:27:54.125820+0000","last_peered":"2024-09-21T10:27:54.125820+0000","last_clean":"2024-09-21T10:27:54.125820+0000","last_became_active":"2024-09-21T10:24:25.831245+0000","last_became_peered":"2024-09-21T10:24:25.831245+0000","last_unstale":"2024-09-21T10:27:54.125820+0000","last_undegraded":"2024-09-21T10:27:54.125820+0000","last_fullsized":"2024-09-21T10:27:54.125820+0000","mapping_epoch":378,"log_start":"386'6300","ondisk_log_start":"386'6300","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7051,"log_dups_size":0,"ondisk_log_size":7051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:30.943590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166133760,"num_objects":2535,"num_object_clones":0,"num_object_copies":7605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13351,"num_write_kb":508352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":509,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.905685+0000","last_change":"2024-09-21T10:25:48.752455+0000","last_active":"2024-09-21T10:27:44.905685+0000","last_peered":"2024-09-21T10:27:44.905685+0000","last_clean":"2024-09-21T10:27:44.905685+0000","last_became_active":"2024-09-21T10:25:48.752056+0000","last_became_peered":"2024-09-21T10:25:48.752056+0000","last_unstale":"2024-09-21T10:27:44.905685+0000","last_undegraded":"2024-09-21T10:27:44.905685+0000","last_fullsized":"2024-09-21T10:27:44.905685+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:15.886186+0000","scrub_duration":0.0018106349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":425,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.936682+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:27:45.936682+0000","last_peered":"2024-09-21T10:27:45.936682+0000","last_clean":"2024-09-21T10:27:45.936682+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:27:45.936682+0000","last_undegraded":"2024-09-21T10:27:45.936682+0000","last_fullsized":"2024-09-21T10:27:45.936682+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":993,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.908938+0000","last_change":"2024-09-21T10:25:48.755967+0000","last_active":"2024-09-21T10:27:44.908938+0000","last_peered":"2024-09-21T10:27:44.908938+0000","last_clean":"2024-09-21T10:27:44.908938+0000","last_became_active":"2024-09-21T10:25:48.755037+0000","last_became_peered":"2024-09-21T10:25:48.755037+0000","last_unstale":"2024-09-21T10:27:44.908938+0000","last_undegraded":"2024-09-21T10:27:44.908938+0000","last_fullsized":"2024-09-21T10:27:44.908938+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:26.414898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":276,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.982681+0000","last_change":"2024-09-21T10:25:48.767902+0000","last_active":"2024-09-21T10:27:56.982681+0000","last_peered":"2024-09-21T10:27:56.982681+0000","last_clean":"2024-09-21T10:27:56.982681+0000","last_became_active":"2024-09-21T10:25:48.765443+0000","last_became_peered":"2024-09-21T10:25:48.765443+0000","last_unstale":"2024-09-21T10:27:56.982681+0000","last_undegraded":"2024-09-21T10:27:56.982681+0000","last_fullsized":"2024-09-21T10:27:56.982681+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:26.622825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"410'13478","reported_seq":13521,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:54.083438+0000","last_change":"2024-09-21T10:24:25.827432+0000","last_active":"2024-09-21T10:27:54.083438+0000","last_peered":"2024-09-21T10:27:54.083438+0000","last_clean":"2024-09-21T10:27:54.083438+0000","last_became_active":"2024-09-21T10:24:25.827047+0000","last_became_peered":"2024-09-21T10:24:25.827047+0000","last_unstale":"2024-09-21T10:27:54.083438+0000","last_undegraded":"2024-09-21T10:27:54.083438+0000","last_fullsized":"2024-09-21T10:27:54.083438+0000","mapping_epoch":378,"log_start":"386'6500","ondisk_log_start":"386'6500","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6978,"log_dups_size":0,"ondisk_log_size":6978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:52.646714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167444504,"num_objects":2556,"num_object_clones":0,"num_object_copies":7668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2556,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13478,"num_write_kb":513025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":646,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.988480+0000","last_change":"2024-09-21T10:24:26.255840+0000","last_active":"2024-09-21T10:27:44.988480+0000","last_peered":"2024-09-21T10:27:44.988480+0000","last_clean":"2024-09-21T10:27:44.988480+0000","last_became_active":"2024-09-21T10:24:24.825549+0000","last_became_peered":"2024-09-21T10:24:24.825549+0000","last_unstale":"2024-09-21T10:27:44.988480+0000","last_undegraded":"2024-09-21T10:27:44.988480+0000","last_fullsized":"2024-09-21T10:27:44.988480+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:26.255753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.255753+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:25.638856+0000","scrub_duration":0.0016223839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":744,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.948606+0000","last_change":"2024-09-21T10:25:48.758590+0000","last_active":"2024-09-21T10:27:45.948606+0000","last_peered":"2024-09-21T10:27:45.948606+0000","last_clean":"2024-09-21T10:27:45.948606+0000","last_became_active":"2024-09-21T10:25:48.758189+0000","last_became_peered":"2024-09-21T10:25:48.758189+0000","last_unstale":"2024-09-21T10:27:45.948606+0000","last_undegraded":"2024-09-21T10:27:45.948606+0000","last_fullsized":"2024-09-21T10:27:45.948606+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:24.126237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1078,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.938033+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:27:45.938033+0000","last_peered":"2024-09-21T10:27:45.938033+0000","last_clean":"2024-09-21T10:27:45.938033+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:27:45.938033+0000","last_undegraded":"2024-09-21T10:27:45.938033+0000","last_fullsized":"2024-09-21T10:27:45.938033+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":740,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.943364+0000","last_change":"2024-09-21T10:25:48.754966+0000","last_active":"2024-09-21T10:27:45.943364+0000","last_peered":"2024-09-21T10:27:45.943364+0000","last_clean":"2024-09-21T10:27:45.943364+0000","last_became_active":"2024-09-21T10:25:48.754152+0000","last_became_peered":"2024-09-21T10:25:48.754152+0000","last_unstale":"2024-09-21T10:27:45.943364+0000","last_undegraded":"2024-09-21T10:27:45.943364+0000","last_fullsized":"2024-09-21T10:27:45.943364+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:33:28.795713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"410'13470","reported_seq":13546,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:54.403570+0000","last_change":"2024-09-21T10:24:25.830908+0000","last_active":"2024-09-21T10:27:54.403570+0000","last_peered":"2024-09-21T10:27:54.403570+0000","last_clean":"2024-09-21T10:27:54.403570+0000","last_became_active":"2024-09-21T10:24:25.830171+0000","last_became_peered":"2024-09-21T10:24:25.830171+0000","last_unstale":"2024-09-21T10:27:54.403570+0000","last_undegraded":"2024-09-21T10:27:54.403570+0000","last_fullsized":"2024-09-21T10:27:54.403570+0000","mapping_epoch":378,"log_start":"386'6600","ondisk_log_start":"386'6600","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6870,"log_dups_size":0,"ondisk_log_size":6870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:00.292732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162267136,"num_objects":2476,"num_object_clones":0,"num_object_copies":7428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13470,"num_write_kb":510272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":712,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.949479+0000","last_change":"2024-09-21T10:24:26.926717+0000","last_active":"2024-09-21T10:27:45.949479+0000","last_peered":"2024-09-21T10:27:45.949479+0000","last_clean":"2024-09-21T10:27:45.949479+0000","last_became_active":"2024-09-21T10:24:24.828205+0000","last_became_peered":"2024-09-21T10:24:24.828205+0000","last_unstale":"2024-09-21T10:27:45.949479+0000","last_undegraded":"2024-09-21T10:27:45.949479+0000","last_fullsized":"2024-09-21T10:27:45.949479+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:26.926661+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.926661+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:47.946104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":96,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.981415+0000","last_change":"2024-09-21T10:25:48.762565+0000","last_active":"2024-09-21T10:27:56.981415+0000","last_peered":"2024-09-21T10:27:56.981415+0000","last_clean":"2024-09-21T10:27:56.981415+0000","last_became_active":"2024-09-21T10:25:48.761324+0000","last_became_peered":"2024-09-21T10:25:48.761324+0000","last_unstale":"2024-09-21T10:27:56.981415+0000","last_undegraded":"2024-09-21T10:27:56.981415+0000","last_fullsized":"2024-09-21T10:27:56.981415+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:58.885790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":721,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.905350+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:27:44.905350+0000","last_peered":"2024-09-21T10:27:44.905350+0000","last_clean":"2024-09-21T10:27:44.905350+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:27:44.905350+0000","last_undegraded":"2024-09-21T10:27:44.905350+0000","last_fullsized":"2024-09-21T10:27:44.905350+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":383,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.987601+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:27:44.987601+0000","last_peered":"2024-09-21T10:27:44.987601+0000","last_clean":"2024-09-21T10:27:44.987601+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:27:44.987601+0000","last_undegraded":"2024-09-21T10:27:44.987601+0000","last_fullsized":"2024-09-21T10:27:44.987601+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":408,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.927667+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:27:45.927667+0000","last_peered":"2024-09-21T10:27:45.927667+0000","last_clean":"2024-09-21T10:27:45.927667+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:27:45.927667+0000","last_undegraded":"2024-09-21T10:27:45.927667+0000","last_fullsized":"2024-09-21T10:27:45.927667+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":117,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.990574+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:27:44.990574+0000","last_peered":"2024-09-21T10:27:44.990574+0000","last_clean":"2024-09-21T10:27:44.990574+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:27:44.990574+0000","last_undegraded":"2024-09-21T10:27:44.990574+0000","last_fullsized":"2024-09-21T10:27:44.990574+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":111,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.905383+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:27:44.905383+0000","last_peered":"2024-09-21T10:27:44.905383+0000","last_clean":"2024-09-21T10:27:44.905383+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:27:44.905383+0000","last_undegraded":"2024-09-21T10:27:44.905383+0000","last_fullsized":"2024-09-21T10:27:44.905383+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":90,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.908679+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:27:44.908679+0000","last_peered":"2024-09-21T10:27:44.908679+0000","last_clean":"2024-09-21T10:27:44.908679+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:27:44.908679+0000","last_undegraded":"2024-09-21T10:27:44.908679+0000","last_fullsized":"2024-09-21T10:27:44.908679+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":119,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.905412+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:27:44.905412+0000","last_peered":"2024-09-21T10:27:44.905412+0000","last_clean":"2024-09-21T10:27:44.905412+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:27:44.905412+0000","last_undegraded":"2024-09-21T10:27:44.905412+0000","last_fullsized":"2024-09-21T10:27:44.905412+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":652,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.946181+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:27:45.946181+0000","last_peered":"2024-09-21T10:27:45.946181+0000","last_clean":"2024-09-21T10:27:45.946181+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:27:45.946181+0000","last_undegraded":"2024-09-21T10:27:45.946181+0000","last_fullsized":"2024-09-21T10:27:45.946181+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":273,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.981701+0000","last_change":"2024-09-21T10:25:48.765786+0000","last_active":"2024-09-21T10:27:56.981701+0000","last_peered":"2024-09-21T10:27:56.981701+0000","last_clean":"2024-09-21T10:27:56.981701+0000","last_became_active":"2024-09-21T10:25:48.764011+0000","last_became_peered":"2024-09-21T10:25:48.764011+0000","last_unstale":"2024-09-21T10:27:56.981701+0000","last_undegraded":"2024-09-21T10:27:56.981701+0000","last_fullsized":"2024-09-21T10:27:56.981701+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:13.352641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":712,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.943220+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:27:45.943220+0000","last_peered":"2024-09-21T10:27:45.943220+0000","last_clean":"2024-09-21T10:27:45.943220+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:27:45.943220+0000","last_undegraded":"2024-09-21T10:27:45.943220+0000","last_fullsized":"2024-09-21T10:27:45.943220+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":426,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.922859+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:27:45.922859+0000","last_peered":"2024-09-21T10:27:45.922859+0000","last_clean":"2024-09-21T10:27:45.922859+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:27:45.922859+0000","last_undegraded":"2024-09-21T10:27:45.922859+0000","last_fullsized":"2024-09-21T10:27:45.922859+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":739,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.970462+0000","last_change":"2024-09-21T10:25:48.755341+0000","last_active":"2024-09-21T10:27:45.970462+0000","last_peered":"2024-09-21T10:27:45.970462+0000","last_clean":"2024-09-21T10:27:45.970462+0000","last_became_active":"2024-09-21T10:25:48.754620+0000","last_became_peered":"2024-09-21T10:25:48.754620+0000","last_unstale":"2024-09-21T10:27:45.970462+0000","last_undegraded":"2024-09-21T10:27:45.970462+0000","last_fullsized":"2024-09-21T10:27:45.970462+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:57:21.296549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":307,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.981200+0000","last_change":"2024-09-21T10:25:48.765683+0000","last_active":"2024-09-21T10:27:56.981200+0000","last_peered":"2024-09-21T10:27:56.981200+0000","last_clean":"2024-09-21T10:27:56.981200+0000","last_became_active":"2024-09-21T10:25:48.765281+0000","last_became_peered":"2024-09-21T10:25:48.765281+0000","last_unstale":"2024-09-21T10:27:56.981200+0000","last_undegraded":"2024-09-21T10:27:56.981200+0000","last_fullsized":"2024-09-21T10:27:56.981200+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:41.798683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":427,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.936001+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:27:45.936001+0000","last_peered":"2024-09-21T10:27:45.936001+0000","last_clean":"2024-09-21T10:27:45.936001+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:27:45.936001+0000","last_undegraded":"2024-09-21T10:27:45.936001+0000","last_fullsized":"2024-09-21T10:27:45.936001+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":718,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.941076+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:27:45.941076+0000","last_peered":"2024-09-21T10:27:45.941076+0000","last_clean":"2024-09-21T10:27:45.941076+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:27:45.941076+0000","last_undegraded":"2024-09-21T10:27:45.941076+0000","last_fullsized":"2024-09-21T10:27:45.941076+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":415,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.927330+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:27:45.927330+0000","last_peered":"2024-09-21T10:27:45.927330+0000","last_clean":"2024-09-21T10:27:45.927330+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:27:45.927330+0000","last_undegraded":"2024-09-21T10:27:45.927330+0000","last_fullsized":"2024-09-21T10:27:45.927330+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":717,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.944262+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:27:45.944262+0000","last_peered":"2024-09-21T10:27:45.944262+0000","last_clean":"2024-09-21T10:27:45.944262+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:27:45.944262+0000","last_undegraded":"2024-09-21T10:27:45.944262+0000","last_fullsized":"2024-09-21T10:27:45.944262+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":66,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.982785+0000","last_change":"2024-09-21T10:25:48.767385+0000","last_active":"2024-09-21T10:27:56.982785+0000","last_peered":"2024-09-21T10:27:56.982785+0000","last_clean":"2024-09-21T10:27:56.982785+0000","last_became_active":"2024-09-21T10:25:48.764077+0000","last_became_peered":"2024-09-21T10:25:48.764077+0000","last_unstale":"2024-09-21T10:27:56.982785+0000","last_undegraded":"2024-09-21T10:27:56.982785+0000","last_fullsized":"2024-09-21T10:27:56.982785+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:06:51.969987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.988291+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:27:44.988291+0000","last_peered":"2024-09-21T10:27:44.988291+0000","last_clean":"2024-09-21T10:27:44.988291+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:27:44.988291+0000","last_undegraded":"2024-09-21T10:27:44.988291+0000","last_fullsized":"2024-09-21T10:27:44.988291+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":351,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904641+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:27:44.904641+0000","last_peered":"2024-09-21T10:27:44.904641+0000","last_clean":"2024-09-21T10:27:44.904641+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:27:44.904641+0000","last_undegraded":"2024-09-21T10:27:44.904641+0000","last_fullsized":"2024-09-21T10:27:44.904641+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":122,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.987780+0000","last_change":"2024-09-21T10:25:48.753233+0000","last_active":"2024-09-21T10:27:44.987780+0000","last_peered":"2024-09-21T10:27:44.987780+0000","last_clean":"2024-09-21T10:27:44.987780+0000","last_became_active":"2024-09-21T10:25:48.752932+0000","last_became_peered":"2024-09-21T10:25:48.752932+0000","last_unstale":"2024-09-21T10:27:44.987780+0000","last_undegraded":"2024-09-21T10:27:44.987780+0000","last_fullsized":"2024-09-21T10:27:44.987780+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:49.456475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":329,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.904963+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:27:44.904963+0000","last_peered":"2024-09-21T10:27:44.904963+0000","last_clean":"2024-09-21T10:27:44.904963+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:27:44.904963+0000","last_undegraded":"2024-09-21T10:27:44.904963+0000","last_fullsized":"2024-09-21T10:27:44.904963+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":742,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.946474+0000","last_change":"2024-09-21T10:25:48.755262+0000","last_active":"2024-09-21T10:27:45.946474+0000","last_peered":"2024-09-21T10:27:45.946474+0000","last_clean":"2024-09-21T10:27:45.946474+0000","last_became_active":"2024-09-21T10:25:48.754746+0000","last_became_peered":"2024-09-21T10:25:48.754746+0000","last_unstale":"2024-09-21T10:27:45.946474+0000","last_undegraded":"2024-09-21T10:27:45.946474+0000","last_fullsized":"2024-09-21T10:27:45.946474+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:28:17.312418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":694,"reported_epoch":410,"state":"active+clean","last_fresh":"2024-09-21T10:27:45.941562+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:27:45.941562+0000","last_peered":"2024-09-21T10:27:45.941562+0000","last_clean":"2024-09-21T10:27:45.941562+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:27:45.941562+0000","last_undegraded":"2024-09-21T10:27:45.941562+0000","last_fullsized":"2024-09-21T10:27:45.941562+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":107,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-09-21T10:27:44.987259+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:27:44.987259+0000","last_peered":"2024-09-21T10:27:44.987259+0000","last_clean":"2024-09-21T10:27:44.987259+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:27:44.987259+0000","last_undegraded":"2024-09-21T10:27:44.987259+0000","last_fullsized":"2024-09-21T10:27:44.987259+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":99,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.981465+0000","last_change":"2024-09-21T10:25:48.764108+0000","last_active":"2024-09-21T10:27:56.981465+0000","last_peered":"2024-09-21T10:27:56.981465+0000","last_clean":"2024-09-21T10:27:56.981465+0000","last_became_active":"2024-09-21T10:25:48.761361+0000","last_became_peered":"2024-09-21T10:25:48.761361+0000","last_unstale":"2024-09-21T10:27:56.981465+0000","last_undegraded":"2024-09-21T10:27:56.981465+0000","last_fullsized":"2024-09-21T10:27:56.981465+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:49.188529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2603089944,"num_objects":39721,"num_object_clones":0,"num_object_copies":119163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13513,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39721,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":215879,"num_write_kb":8179137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4869,"num_bytes_recovered":319094784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7498444800,"data_stored":7498432584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125579,"ondisk_log_size":125579,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28818,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234459,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2323248,"kb_used_data":1493864,"kb_used_omap":1,"kb_used_meta":829310,"kb_avail":91421904,"statfs":{"total":95995035648,"available":93616029696,"internally_reserved":0,"allocated":1529716736,"data_stored":1527619639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":849213899},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496097,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1192744,"kb_used_data":493856,"kb_used_omap":1,"kb_used_meta":698878,"kb_avail":92552408,"statfs":{"total":95995035648,"available":94773665792,"internally_reserved":0,"allocated":505708544,"data_stored":503619819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":715651527},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569403997,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1940264,"kb_used_data":1420960,"kb_used_omap":7,"kb_used_meta":519224,"kb_avail":91804888,"statfs":{"total":95995035648,"available":94008205312,"internally_reserved":0,"allocated":1455063040,"data_stored":1452970709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":531686296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019370,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1154860,"kb_used_data":497828,"kb_used_omap":1,"kb_used_meta":657022,"kb_avail":92590292,"statfs":{"total":95995035648,"available":94812459008,"internally_reserved":0,"allocated":509775872,"data_stored":507683108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":672790982},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435554,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2237660,"kb_used_data":1317460,"kb_used_omap":5,"kb_used_meta":920186,"kb_avail":91507492,"statfs":{"total":95995035648,"available":93703671808,"internally_reserved":0,"allocated":1349079040,"data_stored":1346998404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":942270504},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,1,1,4,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599081,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263580,"kb_used_data":487764,"kb_used_omap":7,"kb_used_meta":775800,"kb_avail":92481572,"statfs":{"total":95995035648,"available":94701129728,"internally_reserved":0,"allocated":499470336,"data_stored":497389918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":794419661},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762608,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2194904,"kb_used_data":1428688,"kb_used_omap":7,"kb_used_meta":766072,"kb_avail":91550248,"statfs":{"total":95995035648,"available":93747453952,"internally_reserved":0,"allocated":1462976512,"data_stored":1460899108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":784457981},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":389,"seq":1670742278171,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":403932,"kb_used_data":291028,"kb_used_omap":12,"kb_used_meta":112883,"kb_avail":93341220,"statfs":{"total":95995035648,"available":95581409280,"internally_reserved":0,"allocated":298012672,"data_stored":295932006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12595,"internal_metadata":115592909},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11001,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":285802496,"data_stored":285802496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1452019712,"data_stored":1452015640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":491061248,"data_stored":491061248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1323761664,"data_stored":1323761664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":497483776,"data_stored":497483776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1447759872,"data_stored":1447755800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":500105216,"data_stored":500105216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1500450816,"data_stored":1500446744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:28:03.776 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:28:03.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:03.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:03.891 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:03.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:03.992 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:04.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:04.074 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:04.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:04.175 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:04.228 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:04.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:04.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:03 smithi045 ceph-mon[95976]: pgmap v1276: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 13382/106947 objects degraded (12.513%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:04.305 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:03 smithi032 ceph-mon[105748]: pgmap v1276: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 13382/106947 objects degraded (12.513%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:04.305 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:04.305 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:03 smithi032 ceph-mon[101299]: pgmap v1276: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 13382/106947 objects degraded (12.513%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:04.305 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:04.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:04.359 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:04.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:04.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:04.454 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:28:04.455 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:28:04.461 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:04.501 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:28:04.501 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:28:04.509 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:04.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:04.541 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:28:04.541 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:28:04.586 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:28:04.587 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:28:04.611 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:04.634 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:28:04.634 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:28:04.643 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:04.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:04.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:04.727 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:28:04.727 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:28:04.745 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:04.760 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:28:04.760 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:28:04.781 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:04.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:04.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:04.791 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:28:04.792 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:28:04.883 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:04.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:04.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:04.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:04.943 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:28:04.943 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:28:05.023 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:05.056 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:05.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:05.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:05.139 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2222028084' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:28:05.139 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2222028084' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:28:05.158 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:05.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:05.193 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:05.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2222028084' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:28:05.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:05.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:05.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:05.434 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:05.473 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:05.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:05.574 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:05.616 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:05.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:05.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:05.718 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:05.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:05.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:05.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:05.886 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:05.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:05.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:05.988 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:06.028 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:06.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:06.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:06.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:06.162 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:06.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:06.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:06.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:06 smithi045 ceph-mon[95976]: pgmap v1277: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 13253/101913 objects degraded (13.004%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:06.307 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:06.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:06.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:06.408 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:06.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:06.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:06.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:06.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:06 smithi032 ceph-mon[105748]: pgmap v1277: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 13253/101913 objects degraded (13.004%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:06.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:06 smithi032 ceph-mon[101299]: pgmap v1277: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 13253/101913 objects degraded (13.004%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:06.546 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:06.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:06.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:06.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:06.684 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:06.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:06.714 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:06.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:06.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:06.859 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:06.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:06.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:06.961 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:06.994 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:06.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:06.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:07.096 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:07.131 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:07.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:07.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:07.233 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:07.270 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:07.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:07 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 13253/101913 objects degraded (13.004%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:07.372 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:07.406 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:07.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:07.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:07 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 13253/101913 objects degraded (13.004%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:07.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:07 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 13253/101913 objects degraded (13.004%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:07.508 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:07.538 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:07.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:07.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:07.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:07.676 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:07.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:07.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:07.827 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:07.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:07.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:07.929 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:07.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:07.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:07.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:08.070 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:08.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:08.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:08.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:08.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:08.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:08.243 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:08.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:08 smithi045 ceph-mon[95976]: pgmap v1278: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 13253/96090 objects degraded (13.792%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:08.344 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:08.384 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:08.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:08.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:08 smithi032 ceph-mon[105748]: pgmap v1278: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 13253/96090 objects degraded (13.792%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:08.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:08 smithi032 ceph-mon[101299]: pgmap v1278: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 13253/96090 objects degraded (13.792%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:08.486 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:08.518 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:08.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:08.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:08.619 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:08.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:08.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:08.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:08.763 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:08.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:08.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:08.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:08.900 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:08.937 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:08.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:08.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:08.949 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:08.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:28:09.038 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:09.082 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:09.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:09.183 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:09.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:09.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:09.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:09.329 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:09.361 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:09.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:09.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:09.463 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:09.500 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:09.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:09.602 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:09.639 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:09.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:09.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:09.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:09.780 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:09.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:09.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:09.881 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:09.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:09.915 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:09.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:28:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:28:09] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.51.0" 2024-09-21T10:28:10.015 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:10.049 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:10.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:10.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:10.151 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:10.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:10.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:10.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:10 smithi045 ceph-mon[95976]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 13253/95358 objects degraded (13.898%); 1.4 MiB/s, 22 objects/s recovering 2024-09-21T10:28:10.289 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:10.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:10.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:10.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:10.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:10.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:10 smithi032 ceph-mon[101299]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 13253/95358 objects degraded (13.898%); 1.4 MiB/s, 22 objects/s recovering 2024-09-21T10:28:10.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:10 smithi032 ceph-mon[105748]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 13253/95358 objects degraded (13.898%); 1.4 MiB/s, 22 objects/s recovering 2024-09-21T10:28:10.469 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:10.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:10.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:10.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:10.609 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:10.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:10.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:10.745 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:10.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:10.847 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:10.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:10.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:10.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:10.982 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:11.011 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:11.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:11.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:11.112 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:11.139 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:11.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:11.240 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:11.278 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:11.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:11.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:11.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:11.411 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:11.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:11.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:11.513 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:11.554 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:11.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:11.655 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:11.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:11.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:11.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:11.800 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:11.842 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:11.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:11.944 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:11.981 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:11.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:11.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:12.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:12.122 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:12.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:12.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:12.265 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:12.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:12 smithi045 ceph-mon[95976]: pgmap v1280: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 13106/90060 objects degraded (14.553%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:28:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:12 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 13106/90060 objects degraded (14.553%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:12.357 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:12 smithi032 ceph-mon[101299]: pgmap v1280: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 13106/90060 objects degraded (14.553%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:28:12.357 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:12 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 13106/90060 objects degraded (14.553%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:12.357 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:12 smithi032 ceph-mon[105748]: pgmap v1280: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 13106/90060 objects degraded (14.553%); 2.1 MiB/s, 33 objects/s recovering 2024-09-21T10:28:12.357 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:12 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 13106/90060 objects degraded (14.553%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:12.368 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:12.403 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:12.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:12.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:12.463 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:12.482 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:12.506 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:12.535 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:12.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:12.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:12.636 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:12.668 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:12.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:12.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:12.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:12.803 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:12.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:12.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:12.905 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:12.943 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:12.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:13.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:13.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:13.084 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:13.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:13.185 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:13.219 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:13.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:13.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:13.320 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:13.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:13.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:13.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:13.464 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:13.497 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:13.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:13.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:13.599 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:13.630 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:13.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:13.732 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:13.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:13.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:13.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:13.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:13.907 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:13.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:13.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:14.009 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:14.051 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:14.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:14.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:14.053 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:14 smithi032 ceph-mon[101299]: pgmap v1281: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 13106/83151 objects degraded (15.762%); 1.4 MiB/s, 22 objects/s recovering 2024-09-21T10:28:14.053 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:14 smithi032 ceph-mon[105748]: pgmap v1281: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 13106/83151 objects degraded (15.762%); 1.4 MiB/s, 22 objects/s recovering 2024-09-21T10:28:14.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:14.185 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:14.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:14.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:14 smithi045 ceph-mon[95976]: pgmap v1281: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 13106/83151 objects degraded (15.762%); 1.4 MiB/s, 22 objects/s recovering 2024-09-21T10:28:14.287 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:14.324 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:14.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:14.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:14.397 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:28:14.426 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:14.454 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:14.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:14.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:14.589 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:14.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:14.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:14.691 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:14.718 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:14.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:14.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:14.819 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:14.853 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:14.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:14.955 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:14.997 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:14.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:15.098 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:15.127 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:15.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:15.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:15.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:15.264 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:15.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:15.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:15.366 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:15.410 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:15.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:15.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:15.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:15.558 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:15.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:15.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:15.659 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:15.694 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:15.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:15.796 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:15.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:15.830 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:15.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:15.930 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:15.965 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:15.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:15.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:16.067 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:16.115 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:16.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:16 smithi032 ceph-mon[105748]: pgmap v1282: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 12951/77685 objects degraded (16.671%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:28:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:16 smithi032 ceph-mon[101299]: pgmap v1282: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 12951/77685 objects degraded (16.671%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:28:16.215 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:16.249 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:16.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:16.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:16.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:16.385 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:16.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:16.487 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:16.518 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:16.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:16.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:16.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:16 smithi045 ceph-mon[95976]: pgmap v1282: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 12951/77685 objects degraded (16.671%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:28:16.620 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:16.655 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:16.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:16.757 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:16.790 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:16.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:16.891 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:16.929 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:16.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:17.031 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:17.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:17.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:17.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:17.174 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:17.211 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:17.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:17.313 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:17.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12951/77685 objects degraded (16.671%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:17.331 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12951/77685 objects degraded (16.671%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:17.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:17.351 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:17.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:17.451 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:17.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:17.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:17.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12951/77685 objects degraded (16.671%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:17.605 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:17.651 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:17.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:17.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:17.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:17.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:17.793 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:17.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:17.894 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:17.924 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:17.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:17.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:18.026 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:18.060 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:18.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:18.079 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:28:18.162 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:18.196 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:18.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:18.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:18.298 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:18.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:18.331 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:18.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:18.432 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:18 smithi032 ceph-mon[101299]: pgmap v1283: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 12951/71352 objects degraded (18.151%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:18 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4088117353' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:18 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:18 smithi032 ceph-mon[105748]: pgmap v1283: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 12951/71352 objects degraded (18.151%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:18 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4088117353' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:18 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:18.472 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:18.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:18.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:18 smithi045 ceph-mon[95976]: pgmap v1283: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 12951/71352 objects degraded (18.151%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:28:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:18 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4088117353' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:18 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:18.574 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:18.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:18.610 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:18.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:18.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:18.744 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:18.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:18.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:18.845 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:18.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:18.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:18.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:18.920 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:28:18.920 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:28:18.982 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:19.024 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:19.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:19.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:19.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:19.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:19.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:28:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:28:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[101299]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T10:28:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:28:19.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:28:19.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[105748]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T10:28:19.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:19.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:19.255 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:19.293 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:19.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:19.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:19.394 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:19.441 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:19.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:19 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:28:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:19 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:28:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:19 smithi045 ceph-mon[95976]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T10:28:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:19.544 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:19.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:19.581 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:19.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:19.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:19.715 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:19.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:19.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:19.817 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:19.850 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:19.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:19.916 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:28:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:28:19] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-21T10:28:19.952 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:19.997 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:19.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:19.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:20.099 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:20.139 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:20.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:20.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:20.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:20 smithi032 ceph-mon[105748]: pgmap v1285: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 12951/70614 objects degraded (18.341%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:28:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:20 smithi032 ceph-mon[101299]: pgmap v1285: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 12951/70614 objects degraded (18.341%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:28:20.241 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:20.278 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:20.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:20.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:20.380 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:20.395 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:28:20.408 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:20.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:20.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:20.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:20 smithi045 ceph-mon[95976]: pgmap v1285: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 12951/70614 objects degraded (18.341%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:28:20.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:20.538 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:20.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:20.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:20.643 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:20.675 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:20.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:20.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:20.777 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:20.814 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:20.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:20.915 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:20.958 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:20.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:20.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:21.060 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:21.101 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:21.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:21.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:21.203 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:21.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:21.258 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:21.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:21.361 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:21.400 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:21.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:21.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:21.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:21.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:21.531 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:21.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:21.632 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:21.677 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:21.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:21.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:21.778 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:21.805 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:21.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:21.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:21.818 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:21.858 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:21.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:21.959 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:21.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:22.060 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:22.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:22.099 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:22.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:22.200 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:22.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:22.237 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:22.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:22.331 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:28:22.331 INFO:teuthology.orchestra.run.smithi032.stdout:{"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 12804/65568 objects degraded (19.528%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1792,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":412,"num_osds":8,"num_up_osds":8,"osd_up_since":1726914347,"num_in_osds":8,"osd_in_since":1726913989,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":21856,"data_bytes":1435244541,"bytes_used":9687408640,"bytes_avail":758272876544,"bytes_total":767960285184,"degraded_objects":12804,"degraded_total":65568,"degraded_ratio":0.19527818448023423,"recovering_objects_per_sec":30,"recovering_bytes_per_sec":1977892,"recovering_keys_per_sec":0,"num_objects_recovered":302,"num_bytes_recovered":19791872,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":815},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi032.fdolcm","status":"up:active","gid":14655}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.32:8443/","prometheus":"http://172.21.15.32:9283/"}},"servicemap":{"epoch":29,"modified":"2024-09-21T10:27:32.636453+0000","services":{}},"progress_events":{}} 2024-09-21T10:28:22.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:22 smithi032 ceph-mon[105748]: pgmap v1286: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 12804/65568 objects degraded (19.528%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:28:22.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12804/65568 objects degraded (19.528%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:22.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:22 smithi032 ceph-mon[101299]: pgmap v1286: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 12804/65568 objects degraded (19.528%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:28:22.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12804/65568 objects degraded (19.528%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:22.338 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:22.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:22.370 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:22.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:22.471 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:22.504 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:22.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:22.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:22 smithi045 ceph-mon[95976]: pgmap v1286: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 12804/65568 objects degraded (19.528%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:28:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12804/65568 objects degraded (19.528%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:22.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:22.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:22.644 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:22.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:22.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:22.791 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:22.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:22.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:22.931 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:22.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:22.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:23.033 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:23.070 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:23.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:23.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:23.171 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:23.211 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:23.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:23.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:28:23.312 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:23.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:23.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:23.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:23.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3372397040' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:28:23.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3372397040' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:28:23.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:23.518 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:23.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:23.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:23.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3372397040' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:28:23.619 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:23.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:23.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:23.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:23.758 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:23.790 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:23.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:23.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:23.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:23.932 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:23.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:23.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:24.033 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:24.070 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:24.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:24.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:24.207 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:24.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:24.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:24.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:24.346 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:24.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:24.447 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:24 smithi032 ceph-mon[105748]: pgmap v1287: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12804/58923 objects degraded (21.730%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:28:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:24 smithi032 ceph-mon[101299]: pgmap v1287: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12804/58923 objects degraded (21.730%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:28:24.489 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:24.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:24.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:24.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:24 smithi045 ceph-mon[95976]: pgmap v1287: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12804/58923 objects degraded (21.730%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:28:24.590 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:24.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:24.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:24.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:24.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:24.758 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:24.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:24.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:24.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:24.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:24.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:24.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:25.035 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:25.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:25.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:25.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:25.166 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:25.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:25.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:25.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:25.304 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:25.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:25.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:25.406 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:25.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:25.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:25.540 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:25.569 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:25.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:25.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:25.671 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:25.710 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:25.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:25.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:25.812 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:25.848 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:25.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:25.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:25.950 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:25.986 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:25.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:26.088 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:26.127 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:26.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:26.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:26.165 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:28:26.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:28:26.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:26.297 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:26.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:26.351 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:26 smithi032 ceph-mon[101299]: pgmap v1288: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 12634/53589 objects degraded (23.576%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:28:26.351 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:26 smithi032 ceph-mon[105748]: pgmap v1288: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 12634/53589 objects degraded (23.576%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:28:26.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:26.432 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:26.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:26.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:26.534 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:26 smithi045 ceph-mon[95976]: pgmap v1288: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 12634/53589 objects degraded (23.576%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:28:26.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:26.575 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:26.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:26.677 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:26.710 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:26.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:26.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:26.811 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:26.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:26.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:26.943 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:26.984 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:26.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:27.086 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:27.121 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:27.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:27.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:27.223 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:27.268 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:27.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:27.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:27.364 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:27 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12634/53589 objects degraded (23.576%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:27.364 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:27 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12634/53589 objects degraded (23.576%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:27.371 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:27.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:27.406 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:27.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:27.507 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:27 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12634/53589 objects degraded (23.576%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:27.536 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:27.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:27.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:27.638 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:27.665 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:27.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:27.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:27.718 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:28:27.718 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:28:27.767 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:27.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:27.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:27.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:27.897 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:27.954 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:27.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:28.056 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:28.097 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:28.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:28.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:28.199 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:28.230 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:28.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:28.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:28.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:28.358 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:28 smithi032 ceph-mon[101299]: pgmap v1289: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 990 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 12634/47379 objects degraded (26.666%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:28:28.358 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:28 smithi032 ceph-mon[105748]: pgmap v1289: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 990 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 12634/47379 objects degraded (26.666%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:28:28.365 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:28.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:28.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:28.467 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:28.513 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:28.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:28.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:28 smithi045 ceph-mon[95976]: pgmap v1289: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 990 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 12634/47379 objects degraded (26.666%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:28:28.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:28.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:28.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:28.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:28.763 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:28.798 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:28.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:28.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:28.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:28:28.900 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:28.901 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:28.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:28.966 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:28.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:29.067 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:29.095 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:29.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:29.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:29.196 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:29.235 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:29.247 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:29.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:29.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:29.312 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:28:29.312 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:28:29.349 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:29.366 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:28:29.366 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:28:29.400 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:28:29.400 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:28:29.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:29.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:29.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:29.435 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:28:29.436 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:28:29.476 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:28:29.477 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:28:29.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:29.516 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:28:29.516 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:28:29.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:29.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:29.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:29.544 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:28:29.544 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:28:29.580 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:28:29.580 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:28:29.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:29.681 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:28:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:28:29] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.51.0" 2024-09-21T10:28:29.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:29.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:29.790 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:29.829 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:29.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:29.930 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:29.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:29.976 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:29.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:30.077 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:30.116 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:30.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:30.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:30.218 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:30.254 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:30.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:30.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:30.356 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:30.390 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:30.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:30.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:30.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:30 smithi032 ceph-mon[101299]: pgmap v1290: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 975 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 12634/46653 objects degraded (27.081%); 1.9 MiB/s, 29 objects/s recovering 2024-09-21T10:28:30.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:30 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2354519824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:28:30.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:30 smithi032 ceph-mon[105748]: pgmap v1290: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 975 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 12634/46653 objects degraded (27.081%); 1.9 MiB/s, 29 objects/s recovering 2024-09-21T10:28:30.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:30 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2354519824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:28:30.492 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:30.528 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:30.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:30.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:30.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:30 smithi045 ceph-mon[95976]: pgmap v1290: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 975 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 12634/46653 objects degraded (27.081%); 1.9 MiB/s, 29 objects/s recovering 2024-09-21T10:28:30.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:30 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2354519824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:28:30.630 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:30.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:30.673 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:30.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:30.774 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:30.809 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:30.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:30.911 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:30.950 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:30.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:30.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:30.986 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:31.052 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:31.080 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:31.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:31.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:31.182 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:31.216 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:31.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:31.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:31.317 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:31.352 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:31.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:31.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:31.453 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:31.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:31.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:31.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:31.583 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:31.610 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:31.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:31.712 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:31.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:31.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:31.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:31.851 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:31.889 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:31.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:31.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:31.991 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:32.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:32.029 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:32.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:32.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:32.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:32.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:32.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:32.309 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:32.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:32.411 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:32.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:32.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:32 smithi032 ceph-mon[101299]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 866 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 12470/41424 objects degraded (30.103%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:28:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:32 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12470/41424 objects degraded (30.103%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:32 smithi032 ceph-mon[105748]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 866 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 12470/41424 objects degraded (30.103%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:28:32.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:32 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12470/41424 objects degraded (30.103%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:32 smithi045 ceph-mon[95976]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 866 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 12470/41424 objects degraded (30.103%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:28:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:32 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12470/41424 objects degraded (30.103%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:32.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:32.589 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:32.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:32.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:32.732 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:32.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:32.833 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:32.867 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:32.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:32.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:32.970 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:33.006 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:33.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:33.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:33.108 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:33.152 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:33.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:33.254 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:33.290 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:33.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:33.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:33.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:33.434 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:33.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:33.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:33.536 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:33.564 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:33.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:33.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:33.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:33.694 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:33.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:33.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:33.796 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:33.835 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:33.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:33.936 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:33.972 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:33.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:34.074 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:34.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:34.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:34.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:34.211 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:34.237 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:34.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:34.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:34.338 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:34.380 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:34 smithi032 ceph-mon[101299]: pgmap v1292: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 722 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 12470/34536 objects degraded (36.107%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T10:28:34.380 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:34.380 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:34 smithi032 ceph-mon[105748]: pgmap v1292: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 722 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 12470/34536 objects degraded (36.107%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T10:28:34.380 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:34.397 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:34.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:34.499 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:34.527 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:34.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:34.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:34.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:34 smithi045 ceph-mon[95976]: pgmap v1292: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 722 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 12470/34536 objects degraded (36.107%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T10:28:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:34.629 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:34.664 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:34.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:34.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:34.766 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:34.792 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:34.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:34.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:34.894 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:34.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:34.930 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:34.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:35.031 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:35.076 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:35.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:35.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:35.178 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:35.224 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:35.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:35.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:35.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:35.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:35.462 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:35.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:35.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:35.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:35.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:35.653 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:35.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:35.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:35.755 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:35.792 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:35.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:35.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:35.894 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:35.931 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:35.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:36.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:36.070 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:36.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:36.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:36.142 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:28:36.171 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:36.199 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:36.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:36.300 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:36.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:36.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:36.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:36.438 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:36 smithi032 ceph-mon[105748]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 608 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 12297/29028 objects degraded (42.363%); 2.0 MiB/s, 42 objects/s recovering 2024-09-21T10:28:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:36 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2354404591' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:28:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:36 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:28:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:36 smithi032 ceph-mon[101299]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 608 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 12297/29028 objects degraded (42.363%); 2.0 MiB/s, 42 objects/s recovering 2024-09-21T10:28:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:36 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2354404591' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:28:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:36 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:28:36.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:36.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:36.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:36.646 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:36 smithi045 ceph-mon[95976]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 608 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 12297/29028 objects degraded (42.363%); 2.0 MiB/s, 42 objects/s recovering 2024-09-21T10:28:36.646 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:36 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2354404591' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:28:36.646 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:36 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:28:36.747 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:36.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:36.787 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:36.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:36.888 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:36.929 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:36.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:37.031 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:37.063 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:37.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:37.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:37.199 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:37.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:37.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:37.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:37.339 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:37.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:37.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:37.441 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:37 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:28:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:37 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:28:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:37 smithi032 ceph-mon[105748]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T10:28:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:37 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12297/29028 objects degraded (42.363%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:37 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:28:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:37 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:28:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:37 smithi032 ceph-mon[101299]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T10:28:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:37 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12297/29028 objects degraded (42.363%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:37.471 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:37.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:37.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:37 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:28:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:37 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:28:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:37 smithi045 ceph-mon[95976]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T10:28:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:37 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12297/29028 objects degraded (42.363%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:37.573 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:37.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:37.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:37.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:37.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:37.742 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:37.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:37.843 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:37.892 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:37.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:37.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:37.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:38.025 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:38.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:38.127 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:38.161 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:38.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:38.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:38.233 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:28:38.234 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:28:38.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:38.302 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:38.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:38.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:38.371 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:38 smithi032 ceph-mon[105748]: pgmap v1295: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 487 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 12297/23229 objects degraded (52.938%); 1.4 MiB/s, 33 objects/s recovering 2024-09-21T10:28:38.371 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:38 smithi032 ceph-mon[101299]: pgmap v1295: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 487 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 12297/23229 objects degraded (52.938%); 1.4 MiB/s, 33 objects/s recovering 2024-09-21T10:28:38.404 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:38.451 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:38.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:38.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:38 smithi045 ceph-mon[95976]: pgmap v1295: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 487 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 12297/23229 objects degraded (52.938%); 1.4 MiB/s, 33 objects/s recovering 2024-09-21T10:28:38.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:38.590 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:38.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:38.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:38.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:38.747 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:38.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:38.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:38.848 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:38.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:38.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:39.017 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:39.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:39.064 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:39.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:39.164 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:39.203 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:39.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:39.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:39.252 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:28:39.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:39.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:39.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:39.384 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:39 smithi032 ceph-mon[105748]: from='client.36153 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:28:39.384 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:28:39.384 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:39 smithi032 ceph-mon[101299]: from='client.36153 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:28:39.384 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:28:39.455 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:39.472 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:39 smithi045 ceph-mon[95976]: from='client.36153 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:28:39.472 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:28:39.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:39.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:39.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:39.612 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:39.651 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:39.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:39.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:39.672 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:28:39.672 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:28:39.672 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:28:39.672 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:28:39.672 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:28:39.672 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:28:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:28:39] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-21T10:28:39.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:39.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:39.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:39.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:39.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:39.913 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:39.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:40.015 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:40.068 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:28:40.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:40.076 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:40.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:40.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:40.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:40.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:40.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:40.311 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:40.351 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:40.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:40.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:40.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:40.456 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:40 smithi045 ceph-mon[95976]: pgmap v1296: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 470 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12297/22419 objects degraded (54.851%); 1.4 MiB/s, 33 objects/s recovering 2024-09-21T10:28:40.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:40 smithi032 ceph-mon[105748]: pgmap v1296: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 470 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12297/22419 objects degraded (54.851%); 1.4 MiB/s, 33 objects/s recovering 2024-09-21T10:28:40.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:40 smithi032 ceph-mon[101299]: pgmap v1296: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 470 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12297/22419 objects degraded (54.851%); 1.4 MiB/s, 33 objects/s recovering 2024-09-21T10:28:40.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:40.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:40.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:40.588 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:40.625 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:40.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:40.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:40.726 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:40.761 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:40.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:40.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:40.862 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:40.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:40.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:40.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:40.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:41.029 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1295,"stamp":"2024-09-21T10:28:36.674637+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":510334973,"num_objects":7743,"num_object_clones":0,"num_object_copies":23229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12297,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7743,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":248793,"num_write_kb":8191762,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6370,"num_bytes_recovered":407836293,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128459,"ondisk_log_size":128459,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7028056,"kb_used_data":1642968,"kb_used_omap":45,"kb_used_meta":5384850,"kb_avail":742933160,"statfs":{"total":767960285184,"available":760763555840,"internally_reserved":0,"allocated":1682399232,"data_stored":1665646599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46417,"internal_metadata":5514087087},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,5,12,10,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-527564800,"num_objects":-8050,"num_object_clones":0,"num_object_copies":-24150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-337,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8050,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":337,"num_bytes_recovered":14352384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006709"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":703,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.099054+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:28:18.099054+0000","last_peered":"2024-09-21T10:28:18.099054+0000","last_clean":"2024-09-21T10:28:18.099054+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:28:18.099054+0000","last_undegraded":"2024-09-21T10:28:18.099054+0000","last_fullsized":"2024-09-21T10:28:18.099054+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":417,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.976811+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:27:56.976811+0000","last_peered":"2024-09-21T10:27:56.976811+0000","last_clean":"2024-09-21T10:27:56.976811+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:27:56.976811+0000","last_undegraded":"2024-09-21T10:27:56.976811+0000","last_fullsized":"2024-09-21T10:27:56.976811+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":113,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.972732+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:27:56.972732+0000","last_peered":"2024-09-21T10:27:56.972732+0000","last_clean":"2024-09-21T10:27:56.972732+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:27:56.972732+0000","last_undegraded":"2024-09-21T10:27:56.972732+0000","last_fullsized":"2024-09-21T10:27:56.972732+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":70,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.088042+0000","last_change":"2024-09-21T10:25:48.765404+0000","last_active":"2024-09-21T10:28:18.088042+0000","last_peered":"2024-09-21T10:28:18.088042+0000","last_clean":"2024-09-21T10:28:18.088042+0000","last_became_active":"2024-09-21T10:25:48.764230+0000","last_became_peered":"2024-09-21T10:25:48.764230+0000","last_unstale":"2024-09-21T10:28:18.088042+0000","last_undegraded":"2024-09-21T10:28:18.088042+0000","last_fullsized":"2024-09-21T10:28:18.088042+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:38.551941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":71,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.088373+0000","last_change":"2024-09-21T10:25:48.764391+0000","last_active":"2024-09-21T10:28:18.088373+0000","last_peered":"2024-09-21T10:28:18.088373+0000","last_clean":"2024-09-21T10:28:18.088373+0000","last_became_active":"2024-09-21T10:25:48.763375+0000","last_became_peered":"2024-09-21T10:25:48.763375+0000","last_unstale":"2024-09-21T10:28:18.088373+0000","last_undegraded":"2024-09-21T10:28:18.088373+0000","last_fullsized":"2024-09-21T10:28:18.088373+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:03.457716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":388,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.976905+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:27:56.976905+0000","last_peered":"2024-09-21T10:27:56.976905+0000","last_clean":"2024-09-21T10:27:56.976905+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:27:56.976905+0000","last_undegraded":"2024-09-21T10:27:56.976905+0000","last_fullsized":"2024-09-21T10:27:56.976905+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":724,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.100058+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:28:18.100058+0000","last_peered":"2024-09-21T10:28:18.100058+0000","last_clean":"2024-09-21T10:28:18.100058+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:28:18.100058+0000","last_undegraded":"2024-09-21T10:28:18.100058+0000","last_fullsized":"2024-09-21T10:28:18.100058+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":484,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.967996+0000","last_change":"2024-09-21T10:25:48.752712+0000","last_active":"2024-09-21T10:27:56.967996+0000","last_peered":"2024-09-21T10:27:56.967996+0000","last_clean":"2024-09-21T10:27:56.967996+0000","last_became_active":"2024-09-21T10:25:48.752204+0000","last_became_peered":"2024-09-21T10:25:48.752204+0000","last_unstale":"2024-09-21T10:27:56.967996+0000","last_undegraded":"2024-09-21T10:27:56.967996+0000","last_fullsized":"2024-09-21T10:27:56.967996+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:40.068520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"412'15370","reported_seq":24598,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:32.179004+0000","last_change":"2024-09-21T10:27:44.942565+0000","last_active":"2024-09-21T10:28:32.179004+0000","last_peered":"2024-09-21T10:28:32.179004+0000","last_clean":"2024-09-21T10:28:32.179004+0000","last_became_active":"2024-09-21T10:27:44.909231+0000","last_became_peered":"2024-09-21T10:27:44.909231+0000","last_unstale":"2024-09-21T10:28:32.179004+0000","last_undegraded":"2024-09-21T10:28:32.179004+0000","last_fullsized":"2024-09-21T10:28:32.179004+0000","mapping_epoch":408,"log_start":"386'6200","ondisk_log_start":"386'6200","created":378,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":9170,"log_dups_size":0,"ondisk_log_size":9170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:13.740223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42336256,"num_objects":646,"num_object_clones":0,"num_object_copies":1938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15370,"num_write_kb":512512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4576,"num_bytes_recovered":299892736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":119,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.080064+0000","last_change":"2024-09-21T10:25:48.755730+0000","last_active":"2024-09-21T10:28:18.080064+0000","last_peered":"2024-09-21T10:28:18.080064+0000","last_clean":"2024-09-21T10:28:18.080064+0000","last_became_active":"2024-09-21T10:25:48.755344+0000","last_became_peered":"2024-09-21T10:25:48.755344+0000","last_unstale":"2024-09-21T10:28:18.080064+0000","last_undegraded":"2024-09-21T10:28:18.080064+0000","last_fullsized":"2024-09-21T10:28:18.080064+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:37.725090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":121,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.971428+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:27:56.971428+0000","last_peered":"2024-09-21T10:27:56.971428+0000","last_clean":"2024-09-21T10:27:56.971428+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:27:56.971428+0000","last_undegraded":"2024-09-21T10:27:56.971428+0000","last_fullsized":"2024-09-21T10:27:56.971428+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"412'15533","reported_seq":15614,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:35.989210+0000","last_change":"2024-09-21T10:24:25.829780+0000","last_active":"2024-09-21T10:28:35.989210+0000","last_peered":"2024-09-21T10:28:35.989210+0000","last_clean":"2024-09-21T10:28:35.989210+0000","last_became_active":"2024-09-21T10:24:25.829509+0000","last_became_peered":"2024-09-21T10:24:25.829509+0000","last_unstale":"2024-09-21T10:28:35.989210+0000","last_undegraded":"2024-09-21T10:28:35.989210+0000","last_fullsized":"2024-09-21T10:28:35.989210+0000","mapping_epoch":378,"log_start":"393'8200","ondisk_log_start":"393'8200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7333,"log_dups_size":0,"ondisk_log_size":7333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:30.108791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27197440,"num_objects":415,"num_object_clones":0,"num_object_copies":1245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15533,"num_write_kb":510336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":91,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.079869+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:28:18.079869+0000","last_peered":"2024-09-21T10:28:18.079869+0000","last_clean":"2024-09-21T10:28:18.079869+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:28:18.079869+0000","last_undegraded":"2024-09-21T10:28:18.079869+0000","last_fullsized":"2024-09-21T10:28:18.079869+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":110,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.087950+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:28:18.087950+0000","last_peered":"2024-09-21T10:28:18.087950+0000","last_clean":"2024-09-21T10:28:18.087950+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:28:18.087950+0000","last_undegraded":"2024-09-21T10:28:18.087950+0000","last_fullsized":"2024-09-21T10:28:18.087950+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"412'15573","reported_seq":15617,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:35.993234+0000","last_change":"2024-09-21T10:24:25.825494+0000","last_active":"2024-09-21T10:28:35.993234+0000","last_peered":"2024-09-21T10:28:35.993234+0000","last_clean":"2024-09-21T10:28:35.993234+0000","last_became_active":"2024-09-21T10:24:25.825196+0000","last_became_peered":"2024-09-21T10:24:25.825196+0000","last_unstale":"2024-09-21T10:28:35.993234+0000","last_undegraded":"2024-09-21T10:28:35.993234+0000","last_fullsized":"2024-09-21T10:28:35.993234+0000","mapping_epoch":378,"log_start":"386'6700","ondisk_log_start":"386'6700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8873,"log_dups_size":0,"ondisk_log_size":8873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:50.985537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29556736,"num_objects":451,"num_object_clones":0,"num_object_copies":1353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15573,"num_write_kb":512768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":412,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.977144+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:27:56.977144+0000","last_peered":"2024-09-21T10:27:56.977144+0000","last_clean":"2024-09-21T10:27:56.977144+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:27:56.977144+0000","last_undegraded":"2024-09-21T10:27:56.977144+0000","last_fullsized":"2024-09-21T10:27:56.977144+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":432,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.967984+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:27:56.967984+0000","last_peered":"2024-09-21T10:27:56.967984+0000","last_clean":"2024-09-21T10:27:56.967984+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:27:56.967984+0000","last_undegraded":"2024-09-21T10:27:56.967984+0000","last_fullsized":"2024-09-21T10:27:56.967984+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"412'15125","reported_seq":15169,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:36.001323+0000","last_change":"2024-09-21T10:24:25.846770+0000","last_active":"2024-09-21T10:28:36.001323+0000","last_peered":"2024-09-21T10:28:36.001323+0000","last_clean":"2024-09-21T10:28:36.001323+0000","last_became_active":"2024-09-21T10:24:25.846530+0000","last_became_peered":"2024-09-21T10:24:25.846530+0000","last_unstale":"2024-09-21T10:28:36.001323+0000","last_undegraded":"2024-09-21T10:28:36.001323+0000","last_fullsized":"2024-09-21T10:28:36.001323+0000","mapping_epoch":378,"log_start":"386'6300","ondisk_log_start":"386'6300","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8825,"log_dups_size":0,"ondisk_log_size":8825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:50.827496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30343168,"num_objects":463,"num_object_clones":0,"num_object_copies":1389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15125,"num_write_kb":498816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":346,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.971184+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:27:56.971184+0000","last_peered":"2024-09-21T10:27:56.971184+0000","last_clean":"2024-09-21T10:27:56.971184+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:27:56.971184+0000","last_undegraded":"2024-09-21T10:27:56.971184+0000","last_fullsized":"2024-09-21T10:27:56.971184+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":261,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.089079+0000","last_change":"2024-09-21T10:25:48.763096+0000","last_active":"2024-09-21T10:28:18.089079+0000","last_peered":"2024-09-21T10:28:18.089079+0000","last_clean":"2024-09-21T10:28:18.089079+0000","last_became_active":"2024-09-21T10:25:48.761327+0000","last_became_peered":"2024-09-21T10:25:48.761327+0000","last_unstale":"2024-09-21T10:28:18.089079+0000","last_undegraded":"2024-09-21T10:28:18.089079+0000","last_fullsized":"2024-09-21T10:28:18.089079+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:23.154333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"412'15572","reported_seq":15650,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:34.645408+0000","last_change":"2024-09-21T10:24:25.846977+0000","last_active":"2024-09-21T10:28:34.645408+0000","last_peered":"2024-09-21T10:28:34.645408+0000","last_clean":"2024-09-21T10:28:34.645408+0000","last_became_active":"2024-09-21T10:24:25.846670+0000","last_became_peered":"2024-09-21T10:24:25.846670+0000","last_unstale":"2024-09-21T10:28:34.645408+0000","last_undegraded":"2024-09-21T10:28:34.645408+0000","last_fullsized":"2024-09-21T10:28:34.645408+0000","mapping_epoch":378,"log_start":"393'8700","ondisk_log_start":"393'8700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6872,"log_dups_size":0,"ondisk_log_size":6872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:28.387249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34078720,"num_objects":520,"num_object_clones":0,"num_object_copies":1560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15572,"num_write_kb":514944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":698,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:28:09.218616+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:28:09.218616+0000","last_peered":"2024-09-21T10:28:09.218616+0000","last_clean":"2024-09-21T10:28:09.218616+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:28:09.218616+0000","last_undegraded":"2024-09-21T10:28:09.218616+0000","last_fullsized":"2024-09-21T10:28:09.218616+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":764,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.099129+0000","last_change":"2024-09-21T10:25:48.755999+0000","last_active":"2024-09-21T10:28:18.099129+0000","last_peered":"2024-09-21T10:28:18.099129+0000","last_clean":"2024-09-21T10:28:18.099129+0000","last_became_active":"2024-09-21T10:25:48.754257+0000","last_became_peered":"2024-09-21T10:25:48.754257+0000","last_unstale":"2024-09-21T10:28:18.099129+0000","last_undegraded":"2024-09-21T10:28:18.099129+0000","last_fullsized":"2024-09-21T10:28:18.099129+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:06.025116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":109,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.980007+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:27:56.980007+0000","last_peered":"2024-09-21T10:27:56.980007+0000","last_clean":"2024-09-21T10:27:56.980007+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:27:56.980007+0000","last_undegraded":"2024-09-21T10:27:56.980007+0000","last_fullsized":"2024-09-21T10:27:56.980007+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"412'15716","reported_seq":15797,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:35.958342+0000","last_change":"2024-09-21T10:24:25.847167+0000","last_active":"2024-09-21T10:28:35.958342+0000","last_peered":"2024-09-21T10:28:35.958342+0000","last_clean":"2024-09-21T10:28:35.958342+0000","last_became_active":"2024-09-21T10:24:25.846878+0000","last_became_peered":"2024-09-21T10:24:25.846878+0000","last_unstale":"2024-09-21T10:28:35.958342+0000","last_undegraded":"2024-09-21T10:28:35.958342+0000","last_fullsized":"2024-09-21T10:28:35.958342+0000","mapping_epoch":378,"log_start":"393'8300","ondisk_log_start":"393'8300","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7416,"log_dups_size":0,"ondisk_log_size":7416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:32.840244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29753344,"num_objects":454,"num_object_clones":0,"num_object_copies":1362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15716,"num_write_kb":517440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":470,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.971381+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:27:56.971381+0000","last_peered":"2024-09-21T10:27:56.971381+0000","last_clean":"2024-09-21T10:27:56.971381+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:27:56.971381+0000","last_undegraded":"2024-09-21T10:27:56.971381+0000","last_fullsized":"2024-09-21T10:27:56.971381+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":422,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.968112+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:27:56.968112+0000","last_peered":"2024-09-21T10:27:56.968112+0000","last_clean":"2024-09-21T10:27:56.968112+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:27:56.968112+0000","last_undegraded":"2024-09-21T10:27:56.968112+0000","last_fullsized":"2024-09-21T10:27:56.968112+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":109,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.980637+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:27:56.980637+0000","last_peered":"2024-09-21T10:27:56.980637+0000","last_clean":"2024-09-21T10:27:56.980637+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:27:56.980637+0000","last_undegraded":"2024-09-21T10:27:56.980637+0000","last_fullsized":"2024-09-21T10:27:56.980637+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"412'15522","reported_seq":15567,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:34.269519+0000","last_change":"2024-09-21T10:24:25.828673+0000","last_active":"2024-09-21T10:28:34.269519+0000","last_peered":"2024-09-21T10:28:34.269519+0000","last_clean":"2024-09-21T10:28:34.269519+0000","last_became_active":"2024-09-21T10:24:25.828433+0000","last_became_peered":"2024-09-21T10:24:25.828433+0000","last_unstale":"2024-09-21T10:28:34.269519+0000","last_undegraded":"2024-09-21T10:28:34.269519+0000","last_fullsized":"2024-09-21T10:28:34.269519+0000","mapping_epoch":378,"log_start":"393'8500","ondisk_log_start":"393'8500","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:07.654019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15522,"num_write_kb":512640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":110,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.087874+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:28:18.087874+0000","last_peered":"2024-09-21T10:28:18.087874+0000","last_clean":"2024-09-21T10:28:18.087874+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:28:18.087874+0000","last_undegraded":"2024-09-21T10:28:18.087874+0000","last_fullsized":"2024-09-21T10:28:18.087874+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":109,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.982112+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:27:56.982112+0000","last_peered":"2024-09-21T10:27:56.982112+0000","last_clean":"2024-09-21T10:27:56.982112+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:27:56.982112+0000","last_undegraded":"2024-09-21T10:27:56.982112+0000","last_fullsized":"2024-09-21T10:27:56.982112+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"412'15595","reported_seq":15675,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:34.626685+0000","last_change":"2024-09-21T10:24:25.847020+0000","last_active":"2024-09-21T10:28:34.626685+0000","last_peered":"2024-09-21T10:28:34.626685+0000","last_clean":"2024-09-21T10:28:34.626685+0000","last_became_active":"2024-09-21T10:24:25.846622+0000","last_became_peered":"2024-09-21T10:24:25.846622+0000","last_unstale":"2024-09-21T10:28:34.626685+0000","last_undegraded":"2024-09-21T10:28:34.626685+0000","last_fullsized":"2024-09-21T10:28:34.626685+0000","mapping_epoch":378,"log_start":"393'8700","ondisk_log_start":"393'8700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6895,"log_dups_size":0,"ondisk_log_size":6895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:54:04.668262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31522816,"num_objects":481,"num_object_clones":0,"num_object_copies":1443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15595,"num_write_kb":514432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":648,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.971005+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:27:56.971005+0000","last_peered":"2024-09-21T10:27:56.971005+0000","last_clean":"2024-09-21T10:27:56.971005+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:27:56.971005+0000","last_undegraded":"2024-09-21T10:27:56.971005+0000","last_fullsized":"2024-09-21T10:27:56.971005+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":732,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.103591+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:28:18.103591+0000","last_peered":"2024-09-21T10:28:18.103591+0000","last_clean":"2024-09-21T10:28:18.103591+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:28:18.103591+0000","last_undegraded":"2024-09-21T10:28:18.103591+0000","last_fullsized":"2024-09-21T10:28:18.103591+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":424,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.092398+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:28:18.092398+0000","last_peered":"2024-09-21T10:28:18.092398+0000","last_clean":"2024-09-21T10:28:18.092398+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:28:18.092398+0000","last_undegraded":"2024-09-21T10:28:18.092398+0000","last_fullsized":"2024-09-21T10:28:18.092398+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"412'15362","reported_seq":18469,"reported_epoch":412,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:28:34.647227+0000","last_change":"2024-09-21T10:27:43.773504+0000","last_active":"2024-09-21T10:28:34.647227+0000","last_peered":"2024-09-21T10:28:34.647227+0000","last_clean":"2024-09-21T10:24:52.081220+0000","last_became_active":"2024-09-21T10:25:50.482809+0000","last_became_peered":"2024-09-21T10:25:50.482809+0000","last_unstale":"2024-09-21T10:28:34.647227+0000","last_undegraded":"2024-09-21T10:25:49.768122+0000","last_fullsized":"2024-09-21T10:25:49.744590+0000","mapping_epoch":390,"log_start":"393'8500","ondisk_log_start":"393'8500","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6862,"log_dups_size":0,"ondisk_log_size":6862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34471936,"num_objects":526,"num_object_clones":0,"num_object_copies":1578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3028,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15362,"num_write_kb":508416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1508,"num_bytes_recovered":91095040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-2502}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":697,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.976505+0000","last_change":"2024-09-21T10:25:48.754725+0000","last_active":"2024-09-21T10:27:56.976505+0000","last_peered":"2024-09-21T10:27:56.976505+0000","last_clean":"2024-09-21T10:27:56.976505+0000","last_became_active":"2024-09-21T10:25:48.754341+0000","last_became_peered":"2024-09-21T10:25:48.754341+0000","last_unstale":"2024-09-21T10:27:56.976505+0000","last_undegraded":"2024-09-21T10:27:56.976505+0000","last_fullsized":"2024-09-21T10:27:56.976505+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:25.191555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:25.191555+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:08.981054+0000","scrub_duration":0.0015044590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":726,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.077001+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:28:18.077001+0000","last_peered":"2024-09-21T10:28:18.077001+0000","last_clean":"2024-09-21T10:28:18.077001+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:28:18.077001+0000","last_undegraded":"2024-09-21T10:28:18.077001+0000","last_fullsized":"2024-09-21T10:28:18.077001+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":663,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.098578+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:28:18.098578+0000","last_peered":"2024-09-21T10:28:18.098578+0000","last_clean":"2024-09-21T10:28:18.098578+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:28:18.098578+0000","last_undegraded":"2024-09-21T10:28:18.098578+0000","last_fullsized":"2024-09-21T10:28:18.098578+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":389,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:28:04.062755+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:28:04.062755+0000","last_peered":"2024-09-21T10:28:04.062755+0000","last_clean":"2024-09-21T10:28:04.062755+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:28:04.062755+0000","last_undegraded":"2024-09-21T10:28:04.062755+0000","last_fullsized":"2024-09-21T10:28:04.062755+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"412'15304","reported_seq":15365,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:28:35.958325+0000","last_change":"2024-09-21T10:25:50.516299+0000","last_active":"2024-09-21T10:28:35.958325+0000","last_peered":"2024-09-21T10:28:35.958325+0000","last_clean":"2024-09-21T10:24:52.080442+0000","last_became_active":"2024-09-21T10:25:50.497794+0000","last_became_peered":"2024-09-21T10:25:50.497794+0000","last_unstale":"2024-09-21T10:28:35.958325+0000","last_undegraded":"2024-09-21T10:25:49.773994+0000","last_fullsized":"2024-09-21T10:25:49.744196+0000","mapping_epoch":390,"log_start":"386'6400","ondisk_log_start":"386'6400","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8904,"log_dups_size":0,"ondisk_log_size":8904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:02.445442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27918336,"num_objects":426,"num_object_clones":0,"num_object_copies":1278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4583,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15304,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":-4157}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":681,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.092122+0000","last_change":"2024-09-21T10:24:27.359111+0000","last_active":"2024-09-21T10:28:18.092122+0000","last_peered":"2024-09-21T10:28:18.092122+0000","last_clean":"2024-09-21T10:28:18.092122+0000","last_became_active":"2024-09-21T10:24:24.825850+0000","last_became_peered":"2024-09-21T10:24:24.825850+0000","last_unstale":"2024-09-21T10:28:18.092122+0000","last_undegraded":"2024-09-21T10:28:18.092122+0000","last_fullsized":"2024-09-21T10:28:18.092122+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:27.359065+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:24:27.359065+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:06.871319+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1069,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.976510+0000","last_change":"2024-09-21T10:25:48.755667+0000","last_active":"2024-09-21T10:27:56.976510+0000","last_peered":"2024-09-21T10:27:56.976510+0000","last_clean":"2024-09-21T10:27:56.976510+0000","last_became_active":"2024-09-21T10:25:48.755060+0000","last_became_peered":"2024-09-21T10:25:48.755060+0000","last_unstale":"2024-09-21T10:27:56.976510+0000","last_undegraded":"2024-09-21T10:27:56.976510+0000","last_fullsized":"2024-09-21T10:27:56.976510+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:09.230030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":263,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.085655+0000","last_change":"2024-09-21T10:25:48.754659+0000","last_active":"2024-09-21T10:28:18.085655+0000","last_peered":"2024-09-21T10:28:18.085655+0000","last_clean":"2024-09-21T10:28:18.085655+0000","last_became_active":"2024-09-21T10:25:48.754230+0000","last_became_peered":"2024-09-21T10:25:48.754230+0000","last_unstale":"2024-09-21T10:28:18.085655+0000","last_undegraded":"2024-09-21T10:28:18.085655+0000","last_fullsized":"2024-09-21T10:28:18.085655+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:12.555868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":70,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.088154+0000","last_change":"2024-09-21T10:25:48.761249+0000","last_active":"2024-09-21T10:28:18.088154+0000","last_peered":"2024-09-21T10:28:18.088154+0000","last_clean":"2024-09-21T10:28:18.088154+0000","last_became_active":"2024-09-21T10:25:48.759359+0000","last_became_peered":"2024-09-21T10:25:48.759359+0000","last_unstale":"2024-09-21T10:28:18.088154+0000","last_undegraded":"2024-09-21T10:28:18.088154+0000","last_fullsized":"2024-09-21T10:28:18.088154+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:33.510936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"412'15598","reported_seq":15642,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:35.970951+0000","last_change":"2024-09-21T10:24:25.827798+0000","last_active":"2024-09-21T10:28:35.970951+0000","last_peered":"2024-09-21T10:28:35.970951+0000","last_clean":"2024-09-21T10:28:35.970951+0000","last_became_active":"2024-09-21T10:24:25.827230+0000","last_became_peered":"2024-09-21T10:24:25.827230+0000","last_unstale":"2024-09-21T10:28:35.970951+0000","last_undegraded":"2024-09-21T10:28:35.970951+0000","last_fullsized":"2024-09-21T10:28:35.970951+0000","mapping_epoch":378,"log_start":"386'6700","ondisk_log_start":"386'6700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8898,"log_dups_size":0,"ondisk_log_size":8898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:28:46.430403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29753344,"num_objects":454,"num_object_clones":0,"num_object_copies":1362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15598,"num_write_kb":513664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":713,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.977079+0000","last_change":"2024-09-21T10:25:48.755509+0000","last_active":"2024-09-21T10:27:56.977079+0000","last_peered":"2024-09-21T10:27:56.977079+0000","last_clean":"2024-09-21T10:27:56.977079+0000","last_became_active":"2024-09-21T10:25:48.754558+0000","last_became_peered":"2024-09-21T10:25:48.754558+0000","last_unstale":"2024-09-21T10:27:56.977079+0000","last_undegraded":"2024-09-21T10:27:56.977079+0000","last_fullsized":"2024-09-21T10:27:56.977079+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:11.262503+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1264,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.087754+0000","last_change":"2024-09-21T10:25:48.753359+0000","last_active":"2024-09-21T10:28:18.087754+0000","last_peered":"2024-09-21T10:28:18.087754+0000","last_clean":"2024-09-21T10:28:18.087754+0000","last_became_active":"2024-09-21T10:25:48.752653+0000","last_became_peered":"2024-09-21T10:25:48.752653+0000","last_unstale":"2024-09-21T10:28:18.087754+0000","last_undegraded":"2024-09-21T10:28:18.087754+0000","last_fullsized":"2024-09-21T10:28:18.087754+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:55.095131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":411,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.091681+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:28:18.091681+0000","last_peered":"2024-09-21T10:28:18.091681+0000","last_clean":"2024-09-21T10:28:18.091681+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:28:18.091681+0000","last_undegraded":"2024-09-21T10:28:18.091681+0000","last_fullsized":"2024-09-21T10:28:18.091681+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":452,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.092532+0000","last_change":"2024-09-21T10:25:48.766755+0000","last_active":"2024-09-21T10:28:18.092532+0000","last_peered":"2024-09-21T10:28:18.092532+0000","last_clean":"2024-09-21T10:28:18.092532+0000","last_became_active":"2024-09-21T10:25:48.766298+0000","last_became_peered":"2024-09-21T10:25:48.766298+0000","last_unstale":"2024-09-21T10:28:18.092532+0000","last_undegraded":"2024-09-21T10:28:18.092532+0000","last_fullsized":"2024-09-21T10:28:18.092532+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:18.788447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"412'15705","reported_seq":15769,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:28:35.985973+0000","last_change":"2024-09-21T10:25:50.515832+0000","last_active":"2024-09-21T10:28:35.985973+0000","last_peered":"2024-09-21T10:28:35.985973+0000","last_clean":"2024-09-21T10:24:52.091731+0000","last_became_active":"2024-09-21T10:25:50.504214+0000","last_became_peered":"2024-09-21T10:25:50.504214+0000","last_unstale":"2024-09-21T10:28:35.985973+0000","last_undegraded":"2024-09-21T10:25:49.774199+0000","last_fullsized":"2024-09-21T10:25:49.743948+0000","mapping_epoch":390,"log_start":"386'6800","ondisk_log_start":"386'6800","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8905,"log_dups_size":0,"ondisk_log_size":8905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:49.426271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28377088,"num_objects":433,"num_object_clones":0,"num_object_copies":1299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15705,"num_write_kb":516416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":-4253}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":664,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.077273+0000","last_change":"2024-09-21T10:25:48.757672+0000","last_active":"2024-09-21T10:28:18.077273+0000","last_peered":"2024-09-21T10:28:18.077273+0000","last_clean":"2024-09-21T10:28:18.077273+0000","last_became_active":"2024-09-21T10:25:48.757296+0000","last_became_peered":"2024-09-21T10:25:48.757296+0000","last_unstale":"2024-09-21T10:28:18.077273+0000","last_undegraded":"2024-09-21T10:28:18.077273+0000","last_fullsized":"2024-09-21T10:28:18.077273+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:31.244809+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:31.244809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:32.654430+0000","scrub_duration":0.0015828649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1157,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.078255+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:28:18.078255+0000","last_peered":"2024-09-21T10:28:18.078255+0000","last_clean":"2024-09-21T10:28:18.078255+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:28:18.078255+0000","last_undegraded":"2024-09-21T10:28:18.078255+0000","last_fullsized":"2024-09-21T10:28:18.078255+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:28:09.031449+0000","last_change":"2024-09-21T10:25:48.753400+0000","last_active":"2024-09-21T10:28:09.031449+0000","last_peered":"2024-09-21T10:28:09.031449+0000","last_clean":"2024-09-21T10:28:09.031449+0000","last_became_active":"2024-09-21T10:25:48.753088+0000","last_became_peered":"2024-09-21T10:25:48.753088+0000","last_unstale":"2024-09-21T10:28:09.031449+0000","last_undegraded":"2024-09-21T10:28:09.031449+0000","last_fullsized":"2024-09-21T10:28:09.031449+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:49.107566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":411,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.091627+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:28:18.091627+0000","last_peered":"2024-09-21T10:28:18.091627+0000","last_clean":"2024-09-21T10:28:18.091627+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:28:18.091627+0000","last_undegraded":"2024-09-21T10:28:18.091627+0000","last_fullsized":"2024-09-21T10:28:18.091627+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"412'15557","reported_seq":15601,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:35.994532+0000","last_change":"2024-09-21T10:24:25.831730+0000","last_active":"2024-09-21T10:28:35.994532+0000","last_peered":"2024-09-21T10:28:35.994532+0000","last_clean":"2024-09-21T10:28:35.994532+0000","last_became_active":"2024-09-21T10:24:25.831416+0000","last_became_peered":"2024-09-21T10:24:25.831416+0000","last_unstale":"2024-09-21T10:28:35.994532+0000","last_undegraded":"2024-09-21T10:28:35.994532+0000","last_fullsized":"2024-09-21T10:28:35.994532+0000","mapping_epoch":378,"log_start":"386'6700","ondisk_log_start":"386'6700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8857,"log_dups_size":0,"ondisk_log_size":8857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.763992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28508160,"num_objects":435,"num_object_clones":0,"num_object_copies":1305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15557,"num_write_kb":511744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":496,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.968549+0000","last_change":"2024-09-21T10:24:24.828950+0000","last_active":"2024-09-21T10:27:56.968549+0000","last_peered":"2024-09-21T10:27:56.968549+0000","last_clean":"2024-09-21T10:27:56.968549+0000","last_became_active":"2024-09-21T10:24:24.828550+0000","last_became_peered":"2024-09-21T10:24:24.828550+0000","last_unstale":"2024-09-21T10:27:56.968549+0000","last_undegraded":"2024-09-21T10:27:56.968549+0000","last_fullsized":"2024-09-21T10:27:56.968549+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:29.313198+0000","scrub_duration":0.001370488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":362,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.077309+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:28:18.077309+0000","last_peered":"2024-09-21T10:28:18.077309+0000","last_clean":"2024-09-21T10:28:18.077309+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:28:18.077309+0000","last_undegraded":"2024-09-21T10:28:18.077309+0000","last_fullsized":"2024-09-21T10:28:18.077309+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":1031,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.091832+0000","last_change":"2024-09-21T10:25:48.760586+0000","last_active":"2024-09-21T10:28:18.091832+0000","last_peered":"2024-09-21T10:28:18.091832+0000","last_clean":"2024-09-21T10:28:18.091832+0000","last_became_active":"2024-09-21T10:25:48.760042+0000","last_became_peered":"2024-09-21T10:25:48.760042+0000","last_unstale":"2024-09-21T10:28:18.091832+0000","last_undegraded":"2024-09-21T10:28:18.091832+0000","last_fullsized":"2024-09-21T10:28:18.091832+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:41.526062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":447,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.096656+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:28:18.096656+0000","last_peered":"2024-09-21T10:28:18.096656+0000","last_clean":"2024-09-21T10:28:18.096656+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:28:18.096656+0000","last_undegraded":"2024-09-21T10:28:18.096656+0000","last_fullsized":"2024-09-21T10:28:18.096656+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"412'15355","reported_seq":15399,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:34.216919+0000","last_change":"2024-09-21T10:24:25.831693+0000","last_active":"2024-09-21T10:28:34.216919+0000","last_peered":"2024-09-21T10:28:34.216919+0000","last_clean":"2024-09-21T10:28:34.216919+0000","last_became_active":"2024-09-21T10:24:25.831245+0000","last_became_peered":"2024-09-21T10:24:25.831245+0000","last_unstale":"2024-09-21T10:28:34.216919+0000","last_undegraded":"2024-09-21T10:28:34.216919+0000","last_fullsized":"2024-09-21T10:28:34.216919+0000","mapping_epoch":378,"log_start":"393'8300","ondisk_log_start":"393'8300","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:30.943590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34799616,"num_objects":531,"num_object_clones":0,"num_object_copies":1593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15355,"num_write_kb":508352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":511,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.968318+0000","last_change":"2024-09-21T10:25:48.752455+0000","last_active":"2024-09-21T10:27:56.968318+0000","last_peered":"2024-09-21T10:27:56.968318+0000","last_clean":"2024-09-21T10:27:56.968318+0000","last_became_active":"2024-09-21T10:25:48.752056+0000","last_became_peered":"2024-09-21T10:25:48.752056+0000","last_unstale":"2024-09-21T10:27:56.968318+0000","last_undegraded":"2024-09-21T10:27:56.968318+0000","last_fullsized":"2024-09-21T10:27:56.968318+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:15.886186+0000","scrub_duration":0.0018106349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":429,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.091750+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:28:18.091750+0000","last_peered":"2024-09-21T10:28:18.091750+0000","last_clean":"2024-09-21T10:28:18.091750+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:28:18.091750+0000","last_undegraded":"2024-09-21T10:28:18.091750+0000","last_fullsized":"2024-09-21T10:28:18.091750+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":995,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.977260+0000","last_change":"2024-09-21T10:25:48.755967+0000","last_active":"2024-09-21T10:27:56.977260+0000","last_peered":"2024-09-21T10:27:56.977260+0000","last_clean":"2024-09-21T10:27:56.977260+0000","last_became_active":"2024-09-21T10:25:48.755037+0000","last_became_peered":"2024-09-21T10:25:48.755037+0000","last_unstale":"2024-09-21T10:27:56.977260+0000","last_undegraded":"2024-09-21T10:27:56.977260+0000","last_fullsized":"2024-09-21T10:27:56.977260+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:26.414898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":278,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.088793+0000","last_change":"2024-09-21T10:25:48.767902+0000","last_active":"2024-09-21T10:28:18.088793+0000","last_peered":"2024-09-21T10:28:18.088793+0000","last_clean":"2024-09-21T10:28:18.088793+0000","last_became_active":"2024-09-21T10:25:48.765443+0000","last_became_peered":"2024-09-21T10:25:48.765443+0000","last_unstale":"2024-09-21T10:28:18.088793+0000","last_undegraded":"2024-09-21T10:28:18.088793+0000","last_fullsized":"2024-09-21T10:28:18.088793+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:26.622825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"412'15531","reported_seq":15576,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:34.208957+0000","last_change":"2024-09-21T10:24:25.827432+0000","last_active":"2024-09-21T10:28:34.208957+0000","last_peered":"2024-09-21T10:28:34.208957+0000","last_clean":"2024-09-21T10:28:34.208957+0000","last_became_active":"2024-09-21T10:24:25.827047+0000","last_became_peered":"2024-09-21T10:24:25.827047+0000","last_unstale":"2024-09-21T10:28:34.208957+0000","last_undegraded":"2024-09-21T10:28:34.208957+0000","last_fullsized":"2024-09-21T10:28:34.208957+0000","mapping_epoch":378,"log_start":"393'8500","ondisk_log_start":"393'8500","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:52.646714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32899096,"num_objects":503,"num_object_clones":0,"num_object_copies":1509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":503,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15531,"num_write_kb":513025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":648,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.980734+0000","last_change":"2024-09-21T10:24:26.255840+0000","last_active":"2024-09-21T10:27:56.980734+0000","last_peered":"2024-09-21T10:27:56.980734+0000","last_clean":"2024-09-21T10:27:56.980734+0000","last_became_active":"2024-09-21T10:24:24.825549+0000","last_became_peered":"2024-09-21T10:24:24.825549+0000","last_unstale":"2024-09-21T10:27:56.980734+0000","last_undegraded":"2024-09-21T10:27:56.980734+0000","last_fullsized":"2024-09-21T10:27:56.980734+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:26.255753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.255753+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:25.638856+0000","scrub_duration":0.0016223839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":748,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.100227+0000","last_change":"2024-09-21T10:25:48.758590+0000","last_active":"2024-09-21T10:28:18.100227+0000","last_peered":"2024-09-21T10:28:18.100227+0000","last_clean":"2024-09-21T10:28:18.100227+0000","last_became_active":"2024-09-21T10:25:48.758189+0000","last_became_peered":"2024-09-21T10:25:48.758189+0000","last_unstale":"2024-09-21T10:28:18.100227+0000","last_undegraded":"2024-09-21T10:28:18.100227+0000","last_fullsized":"2024-09-21T10:28:18.100227+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:24.126237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1083,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.093322+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:28:18.093322+0000","last_peered":"2024-09-21T10:28:18.093322+0000","last_clean":"2024-09-21T10:28:18.093322+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:28:18.093322+0000","last_undegraded":"2024-09-21T10:28:18.093322+0000","last_fullsized":"2024-09-21T10:28:18.093322+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":744,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.097137+0000","last_change":"2024-09-21T10:25:48.754966+0000","last_active":"2024-09-21T10:28:18.097137+0000","last_peered":"2024-09-21T10:28:18.097137+0000","last_clean":"2024-09-21T10:28:18.097137+0000","last_became_active":"2024-09-21T10:25:48.754152+0000","last_became_peered":"2024-09-21T10:25:48.754152+0000","last_unstale":"2024-09-21T10:28:18.097137+0000","last_undegraded":"2024-09-21T10:28:18.097137+0000","last_fullsized":"2024-09-21T10:28:18.097137+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:33:28.795713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"412'15467","reported_seq":15547,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:34.636279+0000","last_change":"2024-09-21T10:24:25.830908+0000","last_active":"2024-09-21T10:28:34.636279+0000","last_peered":"2024-09-21T10:28:34.636279+0000","last_clean":"2024-09-21T10:28:34.636279+0000","last_became_active":"2024-09-21T10:24:25.830171+0000","last_became_peered":"2024-09-21T10:24:25.830171+0000","last_unstale":"2024-09-21T10:28:34.636279+0000","last_undegraded":"2024-09-21T10:28:34.636279+0000","last_fullsized":"2024-09-21T10:28:34.636279+0000","mapping_epoch":378,"log_start":"393'8600","ondisk_log_start":"393'8600","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6867,"log_dups_size":0,"ondisk_log_size":6867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:00.292732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31391744,"num_objects":479,"num_object_clones":0,"num_object_copies":1437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15467,"num_write_kb":510272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":717,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.098879+0000","last_change":"2024-09-21T10:24:26.926717+0000","last_active":"2024-09-21T10:28:18.098879+0000","last_peered":"2024-09-21T10:28:18.098879+0000","last_clean":"2024-09-21T10:28:18.098879+0000","last_became_active":"2024-09-21T10:24:24.828205+0000","last_became_peered":"2024-09-21T10:24:24.828205+0000","last_unstale":"2024-09-21T10:28:18.098879+0000","last_undegraded":"2024-09-21T10:28:18.098879+0000","last_fullsized":"2024-09-21T10:28:18.098879+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:26.926661+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.926661+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:47.946104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":99,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.088450+0000","last_change":"2024-09-21T10:25:48.762565+0000","last_active":"2024-09-21T10:28:18.088450+0000","last_peered":"2024-09-21T10:28:18.088450+0000","last_clean":"2024-09-21T10:28:18.088450+0000","last_became_active":"2024-09-21T10:25:48.761324+0000","last_became_peered":"2024-09-21T10:25:48.761324+0000","last_unstale":"2024-09-21T10:28:18.088450+0000","last_undegraded":"2024-09-21T10:28:18.088450+0000","last_fullsized":"2024-09-21T10:28:18.088450+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:58.885790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":724,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:28:14.066927+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:28:14.066927+0000","last_peered":"2024-09-21T10:28:14.066927+0000","last_clean":"2024-09-21T10:28:14.066927+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:28:14.066927+0000","last_undegraded":"2024-09-21T10:28:14.066927+0000","last_fullsized":"2024-09-21T10:28:14.066927+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":386,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:28:14.038603+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:28:14.038603+0000","last_peered":"2024-09-21T10:28:14.038603+0000","last_clean":"2024-09-21T10:28:14.038603+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:28:14.038603+0000","last_undegraded":"2024-09-21T10:28:14.038603+0000","last_fullsized":"2024-09-21T10:28:14.038603+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":410,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.980313+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:27:56.980313+0000","last_peered":"2024-09-21T10:27:56.980313+0000","last_clean":"2024-09-21T10:27:56.980313+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:27:56.980313+0000","last_undegraded":"2024-09-21T10:27:56.980313+0000","last_fullsized":"2024-09-21T10:27:56.980313+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":119,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.980087+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:27:56.980087+0000","last_peered":"2024-09-21T10:27:56.980087+0000","last_clean":"2024-09-21T10:27:56.980087+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:27:56.980087+0000","last_undegraded":"2024-09-21T10:27:56.980087+0000","last_fullsized":"2024-09-21T10:27:56.980087+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":113,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.971185+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:27:56.971185+0000","last_peered":"2024-09-21T10:27:56.971185+0000","last_clean":"2024-09-21T10:27:56.971185+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:27:56.971185+0000","last_undegraded":"2024-09-21T10:27:56.971185+0000","last_fullsized":"2024-09-21T10:27:56.971185+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":92,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.977002+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:27:56.977002+0000","last_peered":"2024-09-21T10:27:56.977002+0000","last_clean":"2024-09-21T10:27:56.977002+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:27:56.977002+0000","last_undegraded":"2024-09-21T10:27:56.977002+0000","last_fullsized":"2024-09-21T10:27:56.977002+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":121,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.972587+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:27:56.972587+0000","last_peered":"2024-09-21T10:27:56.972587+0000","last_clean":"2024-09-21T10:27:56.972587+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:27:56.972587+0000","last_undegraded":"2024-09-21T10:27:56.972587+0000","last_fullsized":"2024-09-21T10:27:56.972587+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":656,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.096575+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:28:18.096575+0000","last_peered":"2024-09-21T10:28:18.096575+0000","last_clean":"2024-09-21T10:28:18.096575+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:28:18.096575+0000","last_undegraded":"2024-09-21T10:28:18.096575+0000","last_fullsized":"2024-09-21T10:28:18.096575+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":276,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.087949+0000","last_change":"2024-09-21T10:25:48.765786+0000","last_active":"2024-09-21T10:28:18.087949+0000","last_peered":"2024-09-21T10:28:18.087949+0000","last_clean":"2024-09-21T10:28:18.087949+0000","last_became_active":"2024-09-21T10:25:48.764011+0000","last_became_peered":"2024-09-21T10:25:48.764011+0000","last_unstale":"2024-09-21T10:28:18.087949+0000","last_undegraded":"2024-09-21T10:28:18.087949+0000","last_fullsized":"2024-09-21T10:28:18.087949+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:13.352641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":716,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.097062+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:28:18.097062+0000","last_peered":"2024-09-21T10:28:18.097062+0000","last_clean":"2024-09-21T10:28:18.097062+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:28:18.097062+0000","last_undegraded":"2024-09-21T10:28:18.097062+0000","last_fullsized":"2024-09-21T10:28:18.097062+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":428,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.968629+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:27:56.968629+0000","last_peered":"2024-09-21T10:27:56.968629+0000","last_clean":"2024-09-21T10:27:56.968629+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:27:56.968629+0000","last_undegraded":"2024-09-21T10:27:56.968629+0000","last_fullsized":"2024-09-21T10:27:56.968629+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":744,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.101784+0000","last_change":"2024-09-21T10:25:48.755341+0000","last_active":"2024-09-21T10:28:18.101784+0000","last_peered":"2024-09-21T10:28:18.101784+0000","last_clean":"2024-09-21T10:28:18.101784+0000","last_became_active":"2024-09-21T10:25:48.754620+0000","last_became_peered":"2024-09-21T10:25:48.754620+0000","last_unstale":"2024-09-21T10:28:18.101784+0000","last_undegraded":"2024-09-21T10:28:18.101784+0000","last_fullsized":"2024-09-21T10:28:18.101784+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:57:21.296549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":309,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.088237+0000","last_change":"2024-09-21T10:25:48.765683+0000","last_active":"2024-09-21T10:28:18.088237+0000","last_peered":"2024-09-21T10:28:18.088237+0000","last_clean":"2024-09-21T10:28:18.088237+0000","last_became_active":"2024-09-21T10:25:48.765281+0000","last_became_peered":"2024-09-21T10:25:48.765281+0000","last_unstale":"2024-09-21T10:28:18.088237+0000","last_undegraded":"2024-09-21T10:28:18.088237+0000","last_fullsized":"2024-09-21T10:28:18.088237+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:41.798683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":431,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.091836+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:28:18.091836+0000","last_peered":"2024-09-21T10:28:18.091836+0000","last_clean":"2024-09-21T10:28:18.091836+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:28:18.091836+0000","last_undegraded":"2024-09-21T10:28:18.091836+0000","last_fullsized":"2024-09-21T10:28:18.091836+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":722,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.098238+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:28:18.098238+0000","last_peered":"2024-09-21T10:28:18.098238+0000","last_clean":"2024-09-21T10:28:18.098238+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:28:18.098238+0000","last_undegraded":"2024-09-21T10:28:18.098238+0000","last_fullsized":"2024-09-21T10:28:18.098238+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":417,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.980326+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:27:56.980326+0000","last_peered":"2024-09-21T10:27:56.980326+0000","last_clean":"2024-09-21T10:27:56.980326+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:27:56.980326+0000","last_undegraded":"2024-09-21T10:27:56.980326+0000","last_fullsized":"2024-09-21T10:27:56.980326+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":722,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.088123+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:28:18.088123+0000","last_peered":"2024-09-21T10:28:18.088123+0000","last_clean":"2024-09-21T10:28:18.088123+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:28:18.088123+0000","last_undegraded":"2024-09-21T10:28:18.088123+0000","last_fullsized":"2024-09-21T10:28:18.088123+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":69,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.088974+0000","last_change":"2024-09-21T10:25:48.767385+0000","last_active":"2024-09-21T10:28:18.088974+0000","last_peered":"2024-09-21T10:28:18.088974+0000","last_clean":"2024-09-21T10:28:18.088974+0000","last_became_active":"2024-09-21T10:25:48.764077+0000","last_became_peered":"2024-09-21T10:25:48.764077+0000","last_unstale":"2024-09-21T10:28:18.088974+0000","last_undegraded":"2024-09-21T10:28:18.088974+0000","last_fullsized":"2024-09-21T10:28:18.088974+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:06:51.969987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":109,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.980106+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:27:56.980106+0000","last_peered":"2024-09-21T10:27:56.980106+0000","last_clean":"2024-09-21T10:27:56.980106+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:27:56.980106+0000","last_undegraded":"2024-09-21T10:27:56.980106+0000","last_fullsized":"2024-09-21T10:27:56.980106+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":353,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.971048+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:27:56.971048+0000","last_peered":"2024-09-21T10:27:56.971048+0000","last_clean":"2024-09-21T10:27:56.971048+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:27:56.971048+0000","last_undegraded":"2024-09-21T10:27:56.971048+0000","last_fullsized":"2024-09-21T10:27:56.971048+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":124,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.980201+0000","last_change":"2024-09-21T10:25:48.753233+0000","last_active":"2024-09-21T10:27:56.980201+0000","last_peered":"2024-09-21T10:27:56.980201+0000","last_clean":"2024-09-21T10:27:56.980201+0000","last_became_active":"2024-09-21T10:25:48.752932+0000","last_became_peered":"2024-09-21T10:25:48.752932+0000","last_unstale":"2024-09-21T10:27:56.980201+0000","last_undegraded":"2024-09-21T10:27:56.980201+0000","last_fullsized":"2024-09-21T10:27:56.980201+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:49.456475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":331,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.972663+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:27:56.972663+0000","last_peered":"2024-09-21T10:27:56.972663+0000","last_clean":"2024-09-21T10:27:56.972663+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:27:56.972663+0000","last_undegraded":"2024-09-21T10:27:56.972663+0000","last_fullsized":"2024-09-21T10:27:56.972663+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":747,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.096896+0000","last_change":"2024-09-21T10:25:48.755262+0000","last_active":"2024-09-21T10:28:18.096896+0000","last_peered":"2024-09-21T10:28:18.096896+0000","last_clean":"2024-09-21T10:28:18.096896+0000","last_became_active":"2024-09-21T10:25:48.754746+0000","last_became_peered":"2024-09-21T10:25:48.754746+0000","last_unstale":"2024-09-21T10:28:18.096896+0000","last_undegraded":"2024-09-21T10:28:18.096896+0000","last_fullsized":"2024-09-21T10:28:18.096896+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:28:17.312418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":698,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.098588+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:28:18.098588+0000","last_peered":"2024-09-21T10:28:18.098588+0000","last_clean":"2024-09-21T10:28:18.098588+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:28:18.098588+0000","last_undegraded":"2024-09-21T10:28:18.098588+0000","last_fullsized":"2024-09-21T10:28:18.098588+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":109,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-09-21T10:27:56.979541+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:27:56.979541+0000","last_peered":"2024-09-21T10:27:56.979541+0000","last_clean":"2024-09-21T10:27:56.979541+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:27:56.979541+0000","last_undegraded":"2024-09-21T10:27:56.979541+0000","last_fullsized":"2024-09-21T10:27:56.979541+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":101,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-09-21T10:28:18.087698+0000","last_change":"2024-09-21T10:25:48.764108+0000","last_active":"2024-09-21T10:28:18.087698+0000","last_peered":"2024-09-21T10:28:18.087698+0000","last_clean":"2024-09-21T10:28:18.087698+0000","last_became_active":"2024-09-21T10:25:48.761361+0000","last_became_peered":"2024-09-21T10:25:48.761361+0000","last_unstale":"2024-09-21T10:28:18.087698+0000","last_undegraded":"2024-09-21T10:28:18.087698+0000","last_fullsized":"2024-09-21T10:28:18.087698+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:49.188529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":505544728,"num_objects":7715,"num_object_clones":0,"num_object_copies":23145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12297,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7715,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":247885,"num_write_kb":8179137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6085,"num_bytes_recovered":391053312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1551052800,"data_stored":1551040584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125785,"ondisk_log_size":125785,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28818,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234467,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1160264,"kb_used_data":326848,"kb_used_omap":1,"kb_used_meta":833406,"kb_avail":92584888,"statfs":{"total":95995035648,"available":94806925312,"internally_reserved":0,"allocated":334692352,"data_stored":332587431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":853408203},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496105,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":814976,"kb_used_data":107896,"kb_used_omap":1,"kb_used_meta":707070,"kb_avail":92930176,"statfs":{"total":95995035648,"available":95160500224,"internally_reserved":0,"allocated":110485504,"data_stored":108388955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":724040135},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569404005,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":809024,"kb_used_data":267640,"kb_used_omap":7,"kb_used_meta":541368,"kb_avail":92936128,"statfs":{"total":95995035648,"available":95166595072,"internally_reserved":0,"allocated":274063360,"data_stored":271963205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":554361752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,4,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019378,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":758020,"kb_used_data":113276,"kb_used_omap":1,"kb_used_meta":644734,"kb_avail":92987132,"statfs":{"total":95995035648,"available":95218823168,"internally_reserved":0,"allocated":115994624,"data_stored":113894036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":660208070},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435562,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266100,"kb_used_data":281900,"kb_used_omap":5,"kb_used_meta":984058,"kb_avail":92479052,"statfs":{"total":95995035648,"available":94698549248,"internally_reserved":0,"allocated":288665600,"data_stored":286577140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":1007675432},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,4,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599089,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889588,"kb_used_data":104044,"kb_used_omap":7,"kb_used_meta":785528,"kb_avail":92855564,"statfs":{"total":95995035648,"available":95084097536,"internally_reserved":0,"allocated":106541056,"data_stored":104452814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":804381133},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762616,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1024112,"kb_used_data":262184,"kb_used_omap":7,"kb_used_meta":761912,"kb_avail":92721040,"statfs":{"total":95995035648,"available":94946344960,"internally_reserved":0,"allocated":268476416,"data_stored":266391188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":780198141},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":389,"seq":1670742278178,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305972,"kb_used_data":179180,"kb_used_omap":12,"kb_used_meta":126771,"kb_avail":93439180,"statfs":{"total":95995035648,"available":95681720320,"internally_reserved":0,"allocated":183480320,"data_stored":181391830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12595,"internal_metadata":129814221},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11001,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":161873920,"data_stored":161873920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":256380928,"data_stored":256376856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":99745792,"data_stored":99745792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":262078464,"data_stored":262078464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":103022592,"data_stored":103022592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":261033984,"data_stored":261029912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":103153664,"data_stored":103153664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":303763456,"data_stored":303759384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:28:41.038 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:41.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:41.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:41.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:41.190 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:41.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:41.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:41.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:41.332 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:41.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:41.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:41.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:41.469 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:41.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:41.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:41.570 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:41.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:41.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:41.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:41.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:41.744 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:41.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:41.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:41.846 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:41.890 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:41.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:41.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:41.992 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:42.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:42.031 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:42.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:42.132 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:42.192 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:42.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:42.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:42.330 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:42.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:42.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:42.432 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:42.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:42 smithi032 ceph-mon[105748]: pgmap v1297: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 358 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 12116/17034 objects degraded (71.128%); 352 KiB/s, 35 objects/s recovering 2024-09-21T10:28:42.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:42 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12116/17034 objects degraded (71.128%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:42.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:42 smithi032 ceph-mon[101299]: pgmap v1297: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 358 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 12116/17034 objects degraded (71.128%); 352 KiB/s, 35 objects/s recovering 2024-09-21T10:28:42.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:42 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12116/17034 objects degraded (71.128%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:42.475 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:42.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:42.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:42.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:42 smithi045 ceph-mon[95976]: pgmap v1297: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 358 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 12116/17034 objects degraded (71.128%); 352 KiB/s, 35 objects/s recovering 2024-09-21T10:28:42.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:42 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12116/17034 objects degraded (71.128%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:42.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:42.611 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:42.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:42.712 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:42.764 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:42.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:42.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:42.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:28:42.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:42.895 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:42.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:42.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:42.906 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:42.938 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:42.997 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:43.055 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:43.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:43.155 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:43.195 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:43.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:43.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:43.297 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:43.358 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:43.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:43.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:43.505 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:43.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:43.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:43.607 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:43.648 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:43.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:43.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:43.750 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:43.784 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:43.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:43.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:43.886 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:43.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:43.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:44.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:44.034 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:28:44.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:44.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:44.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:44.178 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:44.205 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:44.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:44.307 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:44.345 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:44.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:44.446 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:44 smithi032 ceph-mon[105748]: pgmap v1298: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 217 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12116/10290 objects degraded (117.745%); 352 KiB/s, 35 objects/s recovering 2024-09-21T10:28:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:44 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:44 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:44 smithi032 ceph-mon[101299]: pgmap v1298: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 217 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12116/10290 objects degraded (117.745%); 352 KiB/s, 35 objects/s recovering 2024-09-21T10:28:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:44 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:44 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:44.501 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:44.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:44.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:44 smithi045 ceph-mon[95976]: pgmap v1298: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 217 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12116/10290 objects degraded (117.745%); 352 KiB/s, 35 objects/s recovering 2024-09-21T10:28:44.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:44 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:44.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:44 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:44.601 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:44.635 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:44.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:44.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:44.664 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Removed 127799 objects 2024-09-21T10:28:44.664 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Clean up completed and total clean up time :160.331 2024-09-21T10:28:44.736 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:44.758 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:44.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:44.859 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:44.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:44.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:44.984 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:45.006 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:45.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:45.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:45.107 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:45.131 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:45.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:45.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:45.233 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:45.255 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:45.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:45.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:45.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:45.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:45.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:45.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:45.481 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:45.503 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:45.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:45.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:45.605 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:45.626 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:45.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:45.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:45.729 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:45.751 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:45.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:45.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:45.853 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:45.877 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:45.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:45.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:45.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:46.003 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:46.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:46.104 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:46.128 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:46.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:46.229 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:46.252 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:46.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:46.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:46.353 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:46.376 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:46.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:46.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:46 smithi032 ceph-mon[101299]: pgmap v1299: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 172 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 12116/8142 objects degraded (148.809%); 0 B/s, 18 objects/s recovering 2024-09-21T10:28:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:46 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:46 smithi032 ceph-mon[105748]: pgmap v1299: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 172 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 12116/8142 objects degraded (148.809%); 0 B/s, 18 objects/s recovering 2024-09-21T10:28:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:46 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:46.479 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:46.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:46.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:46 smithi045 ceph-mon[95976]: pgmap v1299: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 172 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 12116/8142 objects degraded (148.809%); 0 B/s, 18 objects/s recovering 2024-09-21T10:28:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:46.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:46 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:46.603 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:46.628 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:46.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:46.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:46.729 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:46.754 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:46.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:46.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:46.856 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:46.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:46.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:46.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:46.982 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:47.005 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:47.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:47.107 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:47.129 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:47.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:47.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:47.231 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:47.254 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:47.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:47.355 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:47.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:47.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:47.383 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:47 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 12116/8142 objects degraded (148.809%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:47.384 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:47 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 12116/8142 objects degraded (148.809%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:47.473 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:47 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 12116/8142 objects degraded (148.809%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:47.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:47.506 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:47.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:47.608 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:47.635 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:47.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:47.693 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:47.694 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:47.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:47.764 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:47.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:47.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:47.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:47.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:47.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:47.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:47.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:48.014 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:48.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:48.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:48.106 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:28:48.116 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:48.142 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:48.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:48.244 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:48.269 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:48.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:48.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:48.281 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:48 smithi032 ceph-mon[101299]: pgmap v1300: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 16 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 11904/636 objects degraded (1871.698%); 0 B/s, 37 objects/s recovering 2024-09-21T10:28:48.281 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:48 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3097670103' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:48.281 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:48 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:48.281 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:48 smithi032 ceph-mon[105748]: pgmap v1300: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 16 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 11904/636 objects degraded (1871.698%); 0 B/s, 37 objects/s recovering 2024-09-21T10:28:48.281 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:48 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3097670103' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:48.281 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:48 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:48.372 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:48.388 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:48 smithi045 ceph-mon[95976]: pgmap v1300: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 16 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 11904/636 objects degraded (1871.698%); 0 B/s, 37 objects/s recovering 2024-09-21T10:28:48.388 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:48 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3097670103' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:48.388 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:48 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:28:48.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:48.396 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:48.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:48.497 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:48.520 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:48.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:48.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:48.622 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:48.644 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:48.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:48.746 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:48.769 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:48.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:48.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:48.871 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:48.893 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:48.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:48.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:48.995 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:49.020 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:49.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:49.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:49.144 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:49.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:49.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:49.246 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:49.271 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:49.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:49.358 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-09-21T10:28:49.358 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-09-21T10:28:49.373 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[105748]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[101299]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:49.385 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:49.395 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:49.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:49.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:49.496 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:49.522 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:49.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:49.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:49 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:28:49.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:49 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:28:49.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:49 smithi045 ceph-mon[95976]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T10:28:49.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:49.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:28:49.623 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:49.651 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:49.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:49.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:28:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:28:49] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.51.0" 2024-09-21T10:28:49.753 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:49.776 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:49.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:49.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:49.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:49.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:50.005 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:50.028 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:50.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:50.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:50.129 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:50.154 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:50.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:50.256 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:50.280 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:50.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:50.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:50.382 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:50.408 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:50.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:50.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:50 smithi032 ceph-mon[101299]: pgmap v1302: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.9 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 11904/246 objects degraded (4839.024%); 0 B/s, 39 objects/s recovering 2024-09-21T10:28:50.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:50 smithi032 ceph-mon[105748]: pgmap v1302: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.9 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 11904/246 objects degraded (4839.024%); 0 B/s, 39 objects/s recovering 2024-09-21T10:28:50.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:50.533 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:50.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:50.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:50.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:50 smithi045 ceph-mon[95976]: pgmap v1302: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.9 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 11904/246 objects degraded (4839.024%); 0 B/s, 39 objects/s recovering 2024-09-21T10:28:50.635 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:50.660 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:50.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:50.762 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:50.786 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:50.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:50.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:50.888 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:50.911 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:50.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:50.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:51.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:51.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:51.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:51.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:51.159 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:51.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:51.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:51.261 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:51.284 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:51.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:51.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:51.409 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:51.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:51.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:51.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:51.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:51 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:51.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:51.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:51 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:51.510 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:51.523 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:51.523 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:51 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:28:51.538 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:51.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:51.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:51.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:51.664 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:51.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:51.766 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:51.793 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:51.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:51.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:51.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:51.918 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:51.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:52.019 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:52.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:52.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:52.149 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:52.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:52.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:52.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:52.273 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:52.295 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:52.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:52.397 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:52.420 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:52.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:52.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:52.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:52 smithi032 ceph-mon[101299]: pgmap v1303: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 10788/84 objects degraded (12842.857%); 0 B/s, 132 objects/s recovering 2024-09-21T10:28:52.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:52 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 10788/84 objects degraded (12842.857%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:52 smithi032 ceph-mon[105748]: pgmap v1303: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 10788/84 objects degraded (12842.857%); 0 B/s, 132 objects/s recovering 2024-09-21T10:28:52.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:52 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 10788/84 objects degraded (12842.857%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:52.521 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:52.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:52 smithi045 ceph-mon[95976]: pgmap v1303: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 10788/84 objects degraded (12842.857%); 0 B/s, 132 objects/s recovering 2024-09-21T10:28:52.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:52 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 10788/84 objects degraded (12842.857%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:52.544 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:52.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:52.645 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:52.668 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:52.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:52.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:52.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:52.793 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:52.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:52.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:52.895 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:52.917 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:52.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:52.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:53.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:53.044 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:53.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:53.146 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:53.172 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:53.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:53.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:53.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:53.296 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:53.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:53.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:53.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:53.420 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:53.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:53.521 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:53.544 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:53.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:53.646 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:53.668 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:53.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:53.769 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:53.792 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:53.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:53.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:53.893 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:53.916 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:53.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:53.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:54.018 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:54.040 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:54.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:54.142 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:54.165 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:54.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:54.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:54.267 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:54.290 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:54.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:54.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:54.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:54.414 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:54.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:54.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:54.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:54 smithi032 ceph-mon[101299]: pgmap v1304: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 10788/84 objects degraded (12842.857%); 0 B/s, 132 objects/s recovering 2024-09-21T10:28:54.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:54 smithi032 ceph-mon[101299]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T10:28:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:54 smithi032 ceph-mon[105748]: pgmap v1304: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 10788/84 objects degraded (12842.857%); 0 B/s, 132 objects/s recovering 2024-09-21T10:28:54.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:54 smithi032 ceph-mon[105748]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T10:28:54.516 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:54 smithi045 ceph-mon[95976]: pgmap v1304: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 10788/84 objects degraded (12842.857%); 0 B/s, 132 objects/s recovering 2024-09-21T10:28:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:54 smithi045 ceph-mon[95976]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T10:28:54.538 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:54.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:54.640 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:54.663 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:54.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:54.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:54.765 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:54.787 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:54.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:54.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:54.889 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:54.912 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:54.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:54.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:55.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:55.036 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:55.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:55.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:28:55.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:55.160 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:55.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:55.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:55.262 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:55.286 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:55.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:55.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:55.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:55.410 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:55.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:55.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:55.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:55 smithi032 ceph-mon[101299]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T10:28:55.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:55 smithi032 ceph-mon[105748]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T10:28:55.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:55.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:55 smithi045 ceph-mon[95976]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T10:28:55.535 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:55.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:55.636 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:55.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:55.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:55.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:55.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:55.783 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:55.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:55.885 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:55.906 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:28:55.906 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:28:55.907 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:55.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:55.925 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:28:55.925 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:28:55.942 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:28:55.942 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:28:55.959 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:28:55.960 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:28:55.976 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:28:55.976 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:28:55.993 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:28:55.993 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:28:56.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:56.011 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:28:56.011 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:28:56.028 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:28:56.028 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:28:56.032 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:56.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:56.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:56.134 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:56.160 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:56.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:56.262 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:56.284 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:56.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:56.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:56.385 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:56.408 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:56.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:56 smithi032 ceph-mon[101299]: pgmap v1307: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 10788/84 objects degraded (12842.857%); 0 B/s, 169 objects/s recovering 2024-09-21T10:28:56.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1356309177' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:28:56.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:56 smithi032 ceph-mon[105748]: pgmap v1307: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 10788/84 objects degraded (12842.857%); 0 B/s, 169 objects/s recovering 2024-09-21T10:28:56.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1356309177' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:28:56.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:56.533 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:56.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:56.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:56.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:56 smithi045 ceph-mon[95976]: pgmap v1307: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 10788/84 objects degraded (12842.857%); 0 B/s, 169 objects/s recovering 2024-09-21T10:28:56.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1356309177' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:28:56.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:56.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:56.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:56.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:56.758 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:56.782 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:56.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:56.824 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:28:56.883 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:28:56.905 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:56.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:56.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:28:57.007 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:57.029 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:57.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:57.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:57.131 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:57.153 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:57.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:57.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:57.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:57.276 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:57.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:57.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:57.378 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:57.402 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:57.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:57.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:57.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:57 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 8699/84 objects degraded (10355.952%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:57.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:57 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 8699/84 objects degraded (10355.952%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:57.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:57.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:57.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:57.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:57 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 8699/84 objects degraded (10355.952%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:28:57.627 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:57.650 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:57.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:57.751 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:57.773 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:57.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:57.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:57.820 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:28:57.820 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:28:57.874 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:57.897 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:57.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:57.998 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:58.020 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:58.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:58.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:58.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:58.144 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:58.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:58.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:58.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:58.269 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:58.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:58.370 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:58.394 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:58.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:58.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:58 smithi032 ceph-mon[101299]: pgmap v1308: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 8699/84 objects degraded (10355.952%); 0 B/s, 400 objects/s recovering 2024-09-21T10:28:58.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:58 smithi032 ceph-mon[105748]: pgmap v1308: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 8699/84 objects degraded (10355.952%); 0 B/s, 400 objects/s recovering 2024-09-21T10:28:58.494 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:58.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:58.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:58.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:58.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:58 smithi045 ceph-mon[95976]: pgmap v1308: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 8699/84 objects degraded (10355.952%); 0 B/s, 400 objects/s recovering 2024-09-21T10:28:58.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:28:58.640 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:58.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:58.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:28:58.730 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1308,"stamp":"2024-09-21T10:28:56.680527+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8699,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":256508,"num_write_kb":8191762,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9968,"num_bytes_recovered":407836293,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128174,"ondisk_log_size":128174,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5489300,"kb_used_data":55672,"kb_used_omap":45,"kb_used_meta":5433490,"kb_avail":744471916,"statfs":{"total":767960285184,"available":762339241984,"internally_reserved":0,"allocated":57008128,"data_stored":40169869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46417,"internal_metadata":5563894447},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-3473432,"num_objects":-54,"num_object_clones":0,"num_object_copies":-162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3205,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-54,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":54,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3205,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001854"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":711,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.149330+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:28:54.149330+0000","last_peered":"2024-09-21T10:28:54.149330+0000","last_clean":"2024-09-21T10:28:54.149330+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:28:54.149330+0000","last_undegraded":"2024-09-21T10:28:54.149330+0000","last_fullsized":"2024-09-21T10:28:54.149330+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":421,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.700092+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:28:53.700092+0000","last_peered":"2024-09-21T10:28:53.700092+0000","last_clean":"2024-09-21T10:28:53.700092+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:28:53.700092+0000","last_undegraded":"2024-09-21T10:28:53.700092+0000","last_fullsized":"2024-09-21T10:28:53.700092+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.145520+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:28:53.145520+0000","last_peered":"2024-09-21T10:28:53.145520+0000","last_clean":"2024-09-21T10:28:53.145520+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:28:53.145520+0000","last_undegraded":"2024-09-21T10:28:53.145520+0000","last_fullsized":"2024-09-21T10:28:53.145520+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":74,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108683+0000","last_change":"2024-09-21T10:25:48.765404+0000","last_active":"2024-09-21T10:28:48.108683+0000","last_peered":"2024-09-21T10:28:48.108683+0000","last_clean":"2024-09-21T10:28:48.108683+0000","last_became_active":"2024-09-21T10:25:48.764230+0000","last_became_peered":"2024-09-21T10:25:48.764230+0000","last_unstale":"2024-09-21T10:28:48.108683+0000","last_undegraded":"2024-09-21T10:28:48.108683+0000","last_fullsized":"2024-09-21T10:28:48.108683+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:38.551941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":75,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108516+0000","last_change":"2024-09-21T10:25:48.764391+0000","last_active":"2024-09-21T10:28:48.108516+0000","last_peered":"2024-09-21T10:28:48.108516+0000","last_clean":"2024-09-21T10:28:48.108516+0000","last_became_active":"2024-09-21T10:25:48.763375+0000","last_became_peered":"2024-09-21T10:25:48.763375+0000","last_unstale":"2024-09-21T10:28:48.108516+0000","last_undegraded":"2024-09-21T10:28:48.108516+0000","last_fullsized":"2024-09-21T10:28:48.108516+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:03.457716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":392,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.700028+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:28:53.700028+0000","last_peered":"2024-09-21T10:28:53.700028+0000","last_clean":"2024-09-21T10:28:53.700028+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:28:53.700028+0000","last_undegraded":"2024-09-21T10:28:53.700028+0000","last_fullsized":"2024-09-21T10:28:53.700028+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":732,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.149403+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:28:54.149403+0000","last_peered":"2024-09-21T10:28:54.149403+0000","last_clean":"2024-09-21T10:28:54.149403+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:28:54.149403+0000","last_undegraded":"2024-09-21T10:28:54.149403+0000","last_fullsized":"2024-09-21T10:28:54.149403+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":487,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.597897+0000","last_change":"2024-09-21T10:25:48.752712+0000","last_active":"2024-09-21T10:28:48.597897+0000","last_peered":"2024-09-21T10:28:48.597897+0000","last_clean":"2024-09-21T10:28:48.597897+0000","last_became_active":"2024-09-21T10:25:48.752204+0000","last_became_peered":"2024-09-21T10:25:48.752204+0000","last_unstale":"2024-09-21T10:28:48.597897+0000","last_undegraded":"2024-09-21T10:28:48.597897+0000","last_fullsized":"2024-09-21T10:28:48.597897+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:40.068520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"413'16016","reported_seq":25248,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108225+0000","last_change":"2024-09-21T10:27:44.942565+0000","last_active":"2024-09-21T10:28:48.108225+0000","last_peered":"2024-09-21T10:28:48.108225+0000","last_clean":"2024-09-21T10:28:48.108225+0000","last_became_active":"2024-09-21T10:27:44.909231+0000","last_became_peered":"2024-09-21T10:27:44.909231+0000","last_unstale":"2024-09-21T10:28:48.108225+0000","last_undegraded":"2024-09-21T10:28:48.108225+0000","last_fullsized":"2024-09-21T10:28:48.108225+0000","mapping_epoch":408,"log_start":"386'6900","ondisk_log_start":"386'6900","created":378,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":9116,"log_dups_size":0,"ondisk_log_size":9116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:13.740223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16016,"num_write_kb":512512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4576,"num_bytes_recovered":299892736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":123,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.700451+0000","last_change":"2024-09-21T10:25:48.755730+0000","last_active":"2024-09-21T10:28:53.700451+0000","last_peered":"2024-09-21T10:28:53.700451+0000","last_clean":"2024-09-21T10:28:53.700451+0000","last_became_active":"2024-09-21T10:25:48.755344+0000","last_became_peered":"2024-09-21T10:25:48.755344+0000","last_unstale":"2024-09-21T10:28:53.700451+0000","last_undegraded":"2024-09-21T10:28:53.700451+0000","last_fullsized":"2024-09-21T10:28:53.700451+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:37.725090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":126,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.144703+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:28:53.144703+0000","last_peered":"2024-09-21T10:28:53.144703+0000","last_clean":"2024-09-21T10:28:53.144703+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:28:53.144703+0000","last_undegraded":"2024-09-21T10:28:53.144703+0000","last_fullsized":"2024-09-21T10:28:53.144703+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"413'15948","reported_seq":16037,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.891319+0000","last_change":"2024-09-21T10:24:25.829780+0000","last_active":"2024-09-21T10:28:54.891319+0000","last_peered":"2024-09-21T10:28:54.891319+0000","last_clean":"2024-09-21T10:28:54.891319+0000","last_became_active":"2024-09-21T10:24:25.829509+0000","last_became_peered":"2024-09-21T10:24:25.829509+0000","last_unstale":"2024-09-21T10:28:54.891319+0000","last_undegraded":"2024-09-21T10:28:54.891319+0000","last_fullsized":"2024-09-21T10:28:54.891319+0000","mapping_epoch":378,"log_start":"393'8600","ondisk_log_start":"393'8600","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7348,"log_dups_size":0,"ondisk_log_size":7348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:30.108791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15948,"num_write_kb":510336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":95,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.700337+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:28:53.700337+0000","last_peered":"2024-09-21T10:28:53.700337+0000","last_clean":"2024-09-21T10:28:53.700337+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:28:53.700337+0000","last_undegraded":"2024-09-21T10:28:53.700337+0000","last_fullsized":"2024-09-21T10:28:53.700337+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":113,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.101994+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:28:48.101994+0000","last_peered":"2024-09-21T10:28:48.101994+0000","last_clean":"2024-09-21T10:28:48.101994+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:28:48.101994+0000","last_undegraded":"2024-09-21T10:28:48.101994+0000","last_fullsized":"2024-09-21T10:28:48.101994+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"413'16024","reported_seq":16073,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.145857+0000","last_change":"2024-09-21T10:24:25.825494+0000","last_active":"2024-09-21T10:28:53.145857+0000","last_peered":"2024-09-21T10:28:53.145857+0000","last_clean":"2024-09-21T10:28:53.145857+0000","last_became_active":"2024-09-21T10:24:25.825196+0000","last_became_peered":"2024-09-21T10:24:25.825196+0000","last_unstale":"2024-09-21T10:28:53.145857+0000","last_undegraded":"2024-09-21T10:28:53.145857+0000","last_fullsized":"2024-09-21T10:28:53.145857+0000","mapping_epoch":378,"log_start":"387'7200","ondisk_log_start":"387'7200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8824,"log_dups_size":0,"ondisk_log_size":8824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:50.985537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16024,"num_write_kb":512768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":417,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.700360+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:28:53.700360+0000","last_peered":"2024-09-21T10:28:53.700360+0000","last_clean":"2024-09-21T10:28:53.700360+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:28:53.700360+0000","last_undegraded":"2024-09-21T10:28:53.700360+0000","last_fullsized":"2024-09-21T10:28:53.700360+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":435,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.597993+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:28:48.597993+0000","last_peered":"2024-09-21T10:28:48.597993+0000","last_clean":"2024-09-21T10:28:48.597993+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:28:48.597993+0000","last_undegraded":"2024-09-21T10:28:48.597993+0000","last_fullsized":"2024-09-21T10:28:48.597993+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"413'15588","reported_seq":15637,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.145451+0000","last_change":"2024-09-21T10:24:25.846770+0000","last_active":"2024-09-21T10:28:53.145451+0000","last_peered":"2024-09-21T10:28:53.145451+0000","last_clean":"2024-09-21T10:28:53.145451+0000","last_became_active":"2024-09-21T10:24:25.846530+0000","last_became_peered":"2024-09-21T10:24:25.846530+0000","last_unstale":"2024-09-21T10:28:53.145451+0000","last_undegraded":"2024-09-21T10:28:53.145451+0000","last_fullsized":"2024-09-21T10:28:53.145451+0000","mapping_epoch":378,"log_start":"386'6700","ondisk_log_start":"386'6700","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8888,"log_dups_size":0,"ondisk_log_size":8888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:50.827496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15588,"num_write_kb":498816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":350,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.145116+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:28:53.145116+0000","last_peered":"2024-09-21T10:28:53.145116+0000","last_clean":"2024-09-21T10:28:53.145116+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:28:53.145116+0000","last_undegraded":"2024-09-21T10:28:53.145116+0000","last_fullsized":"2024-09-21T10:28:53.145116+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":265,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108672+0000","last_change":"2024-09-21T10:25:48.763096+0000","last_active":"2024-09-21T10:28:48.108672+0000","last_peered":"2024-09-21T10:28:48.108672+0000","last_clean":"2024-09-21T10:28:48.108672+0000","last_became_active":"2024-09-21T10:25:48.761327+0000","last_became_peered":"2024-09-21T10:25:48.761327+0000","last_unstale":"2024-09-21T10:28:48.108672+0000","last_undegraded":"2024-09-21T10:28:48.108672+0000","last_fullsized":"2024-09-21T10:28:48.108672+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:23.154333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"413'16092","reported_seq":16181,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148444+0000","last_change":"2024-09-21T10:24:25.846977+0000","last_active":"2024-09-21T10:28:54.148444+0000","last_peered":"2024-09-21T10:28:54.148444+0000","last_clean":"2024-09-21T10:28:54.148444+0000","last_became_active":"2024-09-21T10:24:25.846670+0000","last_became_peered":"2024-09-21T10:24:25.846670+0000","last_unstale":"2024-09-21T10:28:54.148444+0000","last_undegraded":"2024-09-21T10:28:54.148444+0000","last_fullsized":"2024-09-21T10:28:54.148444+0000","mapping_epoch":378,"log_start":"395'9200","ondisk_log_start":"395'9200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:28.387249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16092,"num_write_kb":514944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":702,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.699850+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:28:53.699850+0000","last_peered":"2024-09-21T10:28:53.699850+0000","last_clean":"2024-09-21T10:28:53.699850+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:28:53.699850+0000","last_undegraded":"2024-09-21T10:28:53.699850+0000","last_fullsized":"2024-09-21T10:28:53.699850+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":773,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148772+0000","last_change":"2024-09-21T10:25:48.755999+0000","last_active":"2024-09-21T10:28:54.148772+0000","last_peered":"2024-09-21T10:28:54.148772+0000","last_clean":"2024-09-21T10:28:54.148772+0000","last_became_active":"2024-09-21T10:25:48.754257+0000","last_became_peered":"2024-09-21T10:25:48.754257+0000","last_unstale":"2024-09-21T10:28:54.148772+0000","last_undegraded":"2024-09-21T10:28:54.148772+0000","last_fullsized":"2024-09-21T10:28:54.148772+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:06.025116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":113,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:49.047388+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:28:49.047388+0000","last_peered":"2024-09-21T10:28:49.047388+0000","last_clean":"2024-09-21T10:28:49.047388+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:28:49.047388+0000","last_undegraded":"2024-09-21T10:28:49.047388+0000","last_fullsized":"2024-09-21T10:28:49.047388+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"413'16170","reported_seq":16259,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.890843+0000","last_change":"2024-09-21T10:24:25.847167+0000","last_active":"2024-09-21T10:28:54.890843+0000","last_peered":"2024-09-21T10:28:54.890843+0000","last_clean":"2024-09-21T10:28:54.890843+0000","last_became_active":"2024-09-21T10:24:25.846878+0000","last_became_peered":"2024-09-21T10:24:25.846878+0000","last_unstale":"2024-09-21T10:28:54.890843+0000","last_undegraded":"2024-09-21T10:28:54.890843+0000","last_fullsized":"2024-09-21T10:28:54.890843+0000","mapping_epoch":378,"log_start":"393'8800","ondisk_log_start":"393'8800","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7370,"log_dups_size":0,"ondisk_log_size":7370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:32.840244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16170,"num_write_kb":517440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":475,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.144610+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:28:53.144610+0000","last_peered":"2024-09-21T10:28:53.144610+0000","last_clean":"2024-09-21T10:28:53.144610+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:28:53.144610+0000","last_undegraded":"2024-09-21T10:28:53.144610+0000","last_fullsized":"2024-09-21T10:28:53.144610+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":425,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.598082+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:28:48.598082+0000","last_peered":"2024-09-21T10:28:48.598082+0000","last_clean":"2024-09-21T10:28:48.598082+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:28:48.598082+0000","last_undegraded":"2024-09-21T10:28:48.598082+0000","last_fullsized":"2024-09-21T10:28:48.598082+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":113,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:49.047408+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:28:49.047408+0000","last_peered":"2024-09-21T10:28:49.047408+0000","last_clean":"2024-09-21T10:28:49.047408+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:28:49.047408+0000","last_undegraded":"2024-09-21T10:28:49.047408+0000","last_fullsized":"2024-09-21T10:28:49.047408+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"413'16020","reported_seq":16069,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.699993+0000","last_change":"2024-09-21T10:24:25.828673+0000","last_active":"2024-09-21T10:28:53.699993+0000","last_peered":"2024-09-21T10:28:53.699993+0000","last_clean":"2024-09-21T10:28:53.699993+0000","last_became_active":"2024-09-21T10:24:25.828433+0000","last_became_peered":"2024-09-21T10:24:25.828433+0000","last_unstale":"2024-09-21T10:28:53.699993+0000","last_undegraded":"2024-09-21T10:28:53.699993+0000","last_fullsized":"2024-09-21T10:28:53.699993+0000","mapping_epoch":378,"log_start":"394'9000","ondisk_log_start":"394'9000","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7020,"log_dups_size":0,"ondisk_log_size":7020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:07.654019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16020,"num_write_kb":512640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":113,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.101844+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:28:48.101844+0000","last_peered":"2024-09-21T10:28:48.101844+0000","last_clean":"2024-09-21T10:28:48.101844+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:28:48.101844+0000","last_undegraded":"2024-09-21T10:28:48.101844+0000","last_fullsized":"2024-09-21T10:28:48.101844+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":113,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.198982+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:28:53.198982+0000","last_peered":"2024-09-21T10:28:53.198982+0000","last_clean":"2024-09-21T10:28:53.198982+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:28:53.198982+0000","last_undegraded":"2024-09-21T10:28:53.198982+0000","last_fullsized":"2024-09-21T10:28:53.198982+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"413'16076","reported_seq":16165,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148789+0000","last_change":"2024-09-21T10:24:25.847020+0000","last_active":"2024-09-21T10:28:54.148789+0000","last_peered":"2024-09-21T10:28:54.148789+0000","last_clean":"2024-09-21T10:28:54.148789+0000","last_became_active":"2024-09-21T10:24:25.846622+0000","last_became_peered":"2024-09-21T10:24:25.846622+0000","last_unstale":"2024-09-21T10:28:54.148789+0000","last_undegraded":"2024-09-21T10:28:54.148789+0000","last_fullsized":"2024-09-21T10:28:54.148789+0000","mapping_epoch":378,"log_start":"395'9200","ondisk_log_start":"395'9200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6876,"log_dups_size":0,"ondisk_log_size":6876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:54:04.668262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16076,"num_write_kb":514432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":653,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.144738+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:28:53.144738+0000","last_peered":"2024-09-21T10:28:53.144738+0000","last_clean":"2024-09-21T10:28:53.144738+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:28:53.144738+0000","last_undegraded":"2024-09-21T10:28:53.144738+0000","last_fullsized":"2024-09-21T10:28:53.144738+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":740,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148631+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:28:54.148631+0000","last_peered":"2024-09-21T10:28:54.148631+0000","last_clean":"2024-09-21T10:28:54.148631+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:28:54.148631+0000","last_undegraded":"2024-09-21T10:28:54.148631+0000","last_fullsized":"2024-09-21T10:28:54.148631+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":433,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.890855+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:28:54.890855+0000","last_peered":"2024-09-21T10:28:54.890855+0000","last_clean":"2024-09-21T10:28:54.890855+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:28:54.890855+0000","last_undegraded":"2024-09-21T10:28:54.890855+0000","last_fullsized":"2024-09-21T10:28:54.890855+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"413'15888","reported_seq":25068,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.191746+0000","last_change":"2024-09-21T10:28:54.191631+0000","last_active":"2024-09-21T10:28:54.191746+0000","last_peered":"2024-09-21T10:28:54.191746+0000","last_clean":"2024-09-21T10:28:54.191746+0000","last_became_active":"2024-09-21T10:28:54.151229+0000","last_became_peered":"2024-09-21T10:28:54.151229+0000","last_unstale":"2024-09-21T10:28:54.191746+0000","last_undegraded":"2024-09-21T10:28:54.191746+0000","last_fullsized":"2024-09-21T10:28:54.191746+0000","mapping_epoch":415,"log_start":"395'9000","ondisk_log_start":"395'9000","created":378,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6888,"log_dups_size":0,"ondisk_log_size":6888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:55.061531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15888,"num_write_kb":508416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4536,"num_bytes_recovered":91095040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":701,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.699199+0000","last_change":"2024-09-21T10:25:48.754725+0000","last_active":"2024-09-21T10:28:53.699199+0000","last_peered":"2024-09-21T10:28:53.699199+0000","last_clean":"2024-09-21T10:28:53.699199+0000","last_became_active":"2024-09-21T10:25:48.754341+0000","last_became_peered":"2024-09-21T10:25:48.754341+0000","last_unstale":"2024-09-21T10:28:53.699199+0000","last_undegraded":"2024-09-21T10:28:53.699199+0000","last_fullsized":"2024-09-21T10:28:53.699199+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:25.191555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:25.191555+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:08.981054+0000","scrub_duration":0.0015044590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":730,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.144812+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:28:53.144812+0000","last_peered":"2024-09-21T10:28:53.144812+0000","last_clean":"2024-09-21T10:28:53.144812+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:28:53.144812+0000","last_undegraded":"2024-09-21T10:28:53.144812+0000","last_fullsized":"2024-09-21T10:28:53.144812+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":672,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148935+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:28:54.148935+0000","last_peered":"2024-09-21T10:28:54.148935+0000","last_clean":"2024-09-21T10:28:54.148935+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:28:54.148935+0000","last_undegraded":"2024-09-21T10:28:54.148935+0000","last_fullsized":"2024-09-21T10:28:54.148935+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":392,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.598063+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:28:48.598063+0000","last_peered":"2024-09-21T10:28:48.598063+0000","last_clean":"2024-09-21T10:28:48.598063+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:28:48.598063+0000","last_undegraded":"2024-09-21T10:28:48.598063+0000","last_fullsized":"2024-09-21T10:28:48.598063+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"413'15730","reported_seq":15795,"reported_epoch":415,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:28:53.145351+0000","last_change":"2024-09-21T10:25:50.516299+0000","last_active":"2024-09-21T10:28:53.145351+0000","last_peered":"2024-09-21T10:28:53.145351+0000","last_clean":"2024-09-21T10:24:52.080442+0000","last_became_active":"2024-09-21T10:25:50.497794+0000","last_became_peered":"2024-09-21T10:25:50.497794+0000","last_unstale":"2024-09-21T10:28:53.145351+0000","last_undegraded":"2024-09-21T10:25:49.773994+0000","last_fullsized":"2024-09-21T10:25:49.744196+0000","mapping_epoch":390,"log_start":"387'6900","ondisk_log_start":"387'6900","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8830,"log_dups_size":0,"ondisk_log_size":8830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:02.445442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4583,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15730,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":-4583}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":689,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.891238+0000","last_change":"2024-09-21T10:24:27.359111+0000","last_active":"2024-09-21T10:28:54.891238+0000","last_peered":"2024-09-21T10:28:54.891238+0000","last_clean":"2024-09-21T10:28:54.891238+0000","last_became_active":"2024-09-21T10:24:24.825850+0000","last_became_peered":"2024-09-21T10:24:24.825850+0000","last_unstale":"2024-09-21T10:28:54.891238+0000","last_undegraded":"2024-09-21T10:28:54.891238+0000","last_fullsized":"2024-09-21T10:28:54.891238+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:27.359065+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:24:27.359065+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:06.871319+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1073,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.699284+0000","last_change":"2024-09-21T10:25:48.755667+0000","last_active":"2024-09-21T10:28:53.699284+0000","last_peered":"2024-09-21T10:28:53.699284+0000","last_clean":"2024-09-21T10:28:53.699284+0000","last_became_active":"2024-09-21T10:25:48.755060+0000","last_became_peered":"2024-09-21T10:25:48.755060+0000","last_unstale":"2024-09-21T10:28:53.699284+0000","last_undegraded":"2024-09-21T10:28:53.699284+0000","last_fullsized":"2024-09-21T10:28:53.699284+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:09.230030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":267,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108034+0000","last_change":"2024-09-21T10:25:48.754659+0000","last_active":"2024-09-21T10:28:48.108034+0000","last_peered":"2024-09-21T10:28:48.108034+0000","last_clean":"2024-09-21T10:28:48.108034+0000","last_became_active":"2024-09-21T10:25:48.754230+0000","last_became_peered":"2024-09-21T10:25:48.754230+0000","last_unstale":"2024-09-21T10:28:48.108034+0000","last_undegraded":"2024-09-21T10:28:48.108034+0000","last_fullsized":"2024-09-21T10:28:48.108034+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:12.555868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":74,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108312+0000","last_change":"2024-09-21T10:25:48.761249+0000","last_active":"2024-09-21T10:28:48.108312+0000","last_peered":"2024-09-21T10:28:48.108312+0000","last_clean":"2024-09-21T10:28:48.108312+0000","last_became_active":"2024-09-21T10:25:48.759359+0000","last_became_peered":"2024-09-21T10:25:48.759359+0000","last_unstale":"2024-09-21T10:28:48.108312+0000","last_undegraded":"2024-09-21T10:28:48.108312+0000","last_fullsized":"2024-09-21T10:28:48.108312+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:33.510936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"413'16052","reported_seq":16101,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.144702+0000","last_change":"2024-09-21T10:24:25.827798+0000","last_active":"2024-09-21T10:28:53.144702+0000","last_peered":"2024-09-21T10:28:53.144702+0000","last_clean":"2024-09-21T10:28:53.144702+0000","last_became_active":"2024-09-21T10:24:25.827230+0000","last_became_peered":"2024-09-21T10:24:25.827230+0000","last_unstale":"2024-09-21T10:28:53.144702+0000","last_undegraded":"2024-09-21T10:28:53.144702+0000","last_fullsized":"2024-09-21T10:28:53.144702+0000","mapping_epoch":378,"log_start":"387'7200","ondisk_log_start":"387'7200","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8852,"log_dups_size":0,"ondisk_log_size":8852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:28:46.430403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16052,"num_write_kb":513664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":717,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.699971+0000","last_change":"2024-09-21T10:25:48.755509+0000","last_active":"2024-09-21T10:28:53.699971+0000","last_peered":"2024-09-21T10:28:53.699971+0000","last_clean":"2024-09-21T10:28:53.699971+0000","last_became_active":"2024-09-21T10:25:48.754558+0000","last_became_peered":"2024-09-21T10:25:48.754558+0000","last_unstale":"2024-09-21T10:28:53.699971+0000","last_undegraded":"2024-09-21T10:28:53.699971+0000","last_fullsized":"2024-09-21T10:28:53.699971+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:11.262503+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1270,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.591797+0000","last_change":"2024-09-21T10:25:48.753359+0000","last_active":"2024-09-21T10:28:53.591797+0000","last_peered":"2024-09-21T10:28:53.591797+0000","last_clean":"2024-09-21T10:28:53.591797+0000","last_became_active":"2024-09-21T10:25:48.752653+0000","last_became_peered":"2024-09-21T10:25:48.752653+0000","last_unstale":"2024-09-21T10:28:53.591797+0000","last_undegraded":"2024-09-21T10:28:53.591797+0000","last_fullsized":"2024-09-21T10:28:53.591797+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:55.095131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":419,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.891065+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:28:54.891065+0000","last_peered":"2024-09-21T10:28:54.891065+0000","last_clean":"2024-09-21T10:28:54.891065+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:28:54.891065+0000","last_undegraded":"2024-09-21T10:28:54.891065+0000","last_fullsized":"2024-09-21T10:28:54.891065+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":460,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.891617+0000","last_change":"2024-09-21T10:25:48.766755+0000","last_active":"2024-09-21T10:28:54.891617+0000","last_peered":"2024-09-21T10:28:54.891617+0000","last_clean":"2024-09-21T10:28:54.891617+0000","last_became_active":"2024-09-21T10:25:48.766298+0000","last_became_peered":"2024-09-21T10:25:48.766298+0000","last_unstale":"2024-09-21T10:28:54.891617+0000","last_undegraded":"2024-09-21T10:28:54.891617+0000","last_fullsized":"2024-09-21T10:28:54.891617+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:18.788447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"413'16138","reported_seq":17347,"reported_epoch":416,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:28:56.034020+0000","last_change":"2024-09-21T10:28:53.089865+0000","last_active":"2024-09-21T10:28:56.034020+0000","last_peered":"2024-09-21T10:28:56.034020+0000","last_clean":"2024-09-21T10:24:52.091731+0000","last_became_active":"2024-09-21T10:25:50.504214+0000","last_became_peered":"2024-09-21T10:25:50.504214+0000","last_unstale":"2024-09-21T10:28:56.034020+0000","last_undegraded":"2024-09-21T10:25:49.774199+0000","last_fullsized":"2024-09-21T10:25:49.743948+0000","mapping_epoch":390,"log_start":"387'7300","ondisk_log_start":"387'7300","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8838,"log_dups_size":0,"ondisk_log_size":8838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:49.426271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4116,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16138,"num_write_kb":516416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":571,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":-4116}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":668,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.148831+0000","last_change":"2024-09-21T10:25:48.757672+0000","last_active":"2024-09-21T10:28:53.148831+0000","last_peered":"2024-09-21T10:28:53.148831+0000","last_clean":"2024-09-21T10:28:53.148831+0000","last_became_active":"2024-09-21T10:25:48.757296+0000","last_became_peered":"2024-09-21T10:25:48.757296+0000","last_unstale":"2024-09-21T10:28:53.148831+0000","last_undegraded":"2024-09-21T10:28:53.148831+0000","last_fullsized":"2024-09-21T10:28:53.148831+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:31.244809+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:31.244809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:32.654430+0000","scrub_duration":0.0015828649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1161,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.145060+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:28:53.145060+0000","last_peered":"2024-09-21T10:28:53.145060+0000","last_clean":"2024-09-21T10:28:53.145060+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:28:53.145060+0000","last_undegraded":"2024-09-21T10:28:53.145060+0000","last_fullsized":"2024-09-21T10:28:53.145060+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":128,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.103118+0000","last_change":"2024-09-21T10:25:48.753400+0000","last_active":"2024-09-21T10:28:48.103118+0000","last_peered":"2024-09-21T10:28:48.103118+0000","last_clean":"2024-09-21T10:28:48.103118+0000","last_became_active":"2024-09-21T10:25:48.753088+0000","last_became_peered":"2024-09-21T10:25:48.753088+0000","last_unstale":"2024-09-21T10:28:48.103118+0000","last_undegraded":"2024-09-21T10:28:48.103118+0000","last_fullsized":"2024-09-21T10:28:48.103118+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:49.107566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":419,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.891322+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:28:54.891322+0000","last_peered":"2024-09-21T10:28:54.891322+0000","last_clean":"2024-09-21T10:28:54.891322+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:28:54.891322+0000","last_undegraded":"2024-09-21T10:28:54.891322+0000","last_fullsized":"2024-09-21T10:28:54.891322+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"413'15992","reported_seq":16041,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.150790+0000","last_change":"2024-09-21T10:24:25.831730+0000","last_active":"2024-09-21T10:28:54.150790+0000","last_peered":"2024-09-21T10:28:54.150790+0000","last_clean":"2024-09-21T10:28:54.150790+0000","last_became_active":"2024-09-21T10:24:25.831416+0000","last_became_peered":"2024-09-21T10:24:25.831416+0000","last_unstale":"2024-09-21T10:28:54.150790+0000","last_undegraded":"2024-09-21T10:28:54.150790+0000","last_fullsized":"2024-09-21T10:28:54.150790+0000","mapping_epoch":378,"log_start":"387'7100","ondisk_log_start":"387'7100","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":8892,"log_dups_size":0,"ondisk_log_size":8892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.763992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15992,"num_write_kb":511744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":500,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.598247+0000","last_change":"2024-09-21T10:24:24.828950+0000","last_active":"2024-09-21T10:28:48.598247+0000","last_peered":"2024-09-21T10:28:48.598247+0000","last_clean":"2024-09-21T10:28:48.598247+0000","last_became_active":"2024-09-21T10:24:24.828550+0000","last_became_peered":"2024-09-21T10:24:24.828550+0000","last_unstale":"2024-09-21T10:28:48.598247+0000","last_undegraded":"2024-09-21T10:28:48.598247+0000","last_fullsized":"2024-09-21T10:28:48.598247+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:29.313198+0000","scrub_duration":0.001370488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":366,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.145391+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:28:53.145391+0000","last_peered":"2024-09-21T10:28:53.145391+0000","last_clean":"2024-09-21T10:28:53.145391+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:28:53.145391+0000","last_undegraded":"2024-09-21T10:28:53.145391+0000","last_fullsized":"2024-09-21T10:28:53.145391+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":1039,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.891486+0000","last_change":"2024-09-21T10:25:48.760586+0000","last_active":"2024-09-21T10:28:54.891486+0000","last_peered":"2024-09-21T10:28:54.891486+0000","last_clean":"2024-09-21T10:28:54.891486+0000","last_became_active":"2024-09-21T10:25:48.760042+0000","last_became_peered":"2024-09-21T10:25:48.760042+0000","last_unstale":"2024-09-21T10:28:54.891486+0000","last_undegraded":"2024-09-21T10:28:54.891486+0000","last_fullsized":"2024-09-21T10:28:54.891486+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:41.526062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":455,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.890766+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:28:54.890766+0000","last_peered":"2024-09-21T10:28:54.890766+0000","last_clean":"2024-09-21T10:28:54.890766+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:28:54.890766+0000","last_undegraded":"2024-09-21T10:28:54.890766+0000","last_fullsized":"2024-09-21T10:28:54.890766+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"413'15886","reported_seq":15935,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.149334+0000","last_change":"2024-09-21T10:24:25.831693+0000","last_active":"2024-09-21T10:28:54.149334+0000","last_peered":"2024-09-21T10:28:54.149334+0000","last_clean":"2024-09-21T10:28:54.149334+0000","last_became_active":"2024-09-21T10:24:25.831245+0000","last_became_peered":"2024-09-21T10:24:25.831245+0000","last_unstale":"2024-09-21T10:28:54.149334+0000","last_undegraded":"2024-09-21T10:28:54.149334+0000","last_fullsized":"2024-09-21T10:28:54.149334+0000","mapping_epoch":378,"log_start":"394'8900","ondisk_log_start":"394'8900","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:30.943590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15886,"num_write_kb":508352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":515,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.598199+0000","last_change":"2024-09-21T10:25:48.752455+0000","last_active":"2024-09-21T10:28:48.598199+0000","last_peered":"2024-09-21T10:28:48.598199+0000","last_clean":"2024-09-21T10:28:48.598199+0000","last_became_active":"2024-09-21T10:25:48.752056+0000","last_became_peered":"2024-09-21T10:25:48.752056+0000","last_unstale":"2024-09-21T10:28:48.598199+0000","last_undegraded":"2024-09-21T10:28:48.598199+0000","last_fullsized":"2024-09-21T10:28:48.598199+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:15.886186+0000","scrub_duration":0.0018106349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":437,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.891434+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:28:54.891434+0000","last_peered":"2024-09-21T10:28:54.891434+0000","last_clean":"2024-09-21T10:28:54.891434+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:28:54.891434+0000","last_undegraded":"2024-09-21T10:28:54.891434+0000","last_fullsized":"2024-09-21T10:28:54.891434+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":999,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.700329+0000","last_change":"2024-09-21T10:25:48.755967+0000","last_active":"2024-09-21T10:28:53.700329+0000","last_peered":"2024-09-21T10:28:53.700329+0000","last_clean":"2024-09-21T10:28:53.700329+0000","last_became_active":"2024-09-21T10:25:48.755037+0000","last_became_peered":"2024-09-21T10:25:48.755037+0000","last_unstale":"2024-09-21T10:28:53.700329+0000","last_undegraded":"2024-09-21T10:28:53.700329+0000","last_fullsized":"2024-09-21T10:28:53.700329+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:26.414898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":282,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108316+0000","last_change":"2024-09-21T10:25:48.767902+0000","last_active":"2024-09-21T10:28:48.108316+0000","last_peered":"2024-09-21T10:28:48.108316+0000","last_clean":"2024-09-21T10:28:48.108316+0000","last_became_active":"2024-09-21T10:25:48.765443+0000","last_became_peered":"2024-09-21T10:25:48.765443+0000","last_unstale":"2024-09-21T10:28:48.108316+0000","last_undegraded":"2024-09-21T10:28:48.108316+0000","last_fullsized":"2024-09-21T10:28:48.108316+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:26.622825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"413'16034","reported_seq":16084,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.149225+0000","last_change":"2024-09-21T10:24:25.827432+0000","last_active":"2024-09-21T10:28:54.149225+0000","last_peered":"2024-09-21T10:28:54.149225+0000","last_clean":"2024-09-21T10:28:54.149225+0000","last_became_active":"2024-09-21T10:24:25.827047+0000","last_became_peered":"2024-09-21T10:24:25.827047+0000","last_unstale":"2024-09-21T10:28:54.149225+0000","last_undegraded":"2024-09-21T10:28:54.149225+0000","last_fullsized":"2024-09-21T10:28:54.149225+0000","mapping_epoch":378,"log_start":"394'9000","ondisk_log_start":"394'9000","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":7034,"log_dups_size":0,"ondisk_log_size":7034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:52.646714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":16034,"num_write_kb":513025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":652,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.102533+0000","last_change":"2024-09-21T10:24:26.255840+0000","last_active":"2024-09-21T10:28:48.102533+0000","last_peered":"2024-09-21T10:28:48.102533+0000","last_clean":"2024-09-21T10:28:48.102533+0000","last_became_active":"2024-09-21T10:24:24.825549+0000","last_became_peered":"2024-09-21T10:24:24.825549+0000","last_unstale":"2024-09-21T10:28:48.102533+0000","last_undegraded":"2024-09-21T10:28:48.102533+0000","last_fullsized":"2024-09-21T10:28:48.102533+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:26.255753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.255753+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:25.638856+0000","scrub_duration":0.0016223839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":756,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.149674+0000","last_change":"2024-09-21T10:25:48.758590+0000","last_active":"2024-09-21T10:28:54.149674+0000","last_peered":"2024-09-21T10:28:54.149674+0000","last_clean":"2024-09-21T10:28:54.149674+0000","last_became_active":"2024-09-21T10:25:48.758189+0000","last_became_peered":"2024-09-21T10:25:48.758189+0000","last_unstale":"2024-09-21T10:28:54.149674+0000","last_undegraded":"2024-09-21T10:28:54.149674+0000","last_fullsized":"2024-09-21T10:28:54.149674+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:24.126237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1091,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.891620+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:28:54.891620+0000","last_peered":"2024-09-21T10:28:54.891620+0000","last_clean":"2024-09-21T10:28:54.891620+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:28:54.891620+0000","last_undegraded":"2024-09-21T10:28:54.891620+0000","last_fullsized":"2024-09-21T10:28:54.891620+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":752,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148432+0000","last_change":"2024-09-21T10:25:48.754966+0000","last_active":"2024-09-21T10:28:54.148432+0000","last_peered":"2024-09-21T10:28:54.148432+0000","last_clean":"2024-09-21T10:28:54.148432+0000","last_became_active":"2024-09-21T10:25:48.754152+0000","last_became_peered":"2024-09-21T10:25:48.754152+0000","last_unstale":"2024-09-21T10:28:54.148432+0000","last_undegraded":"2024-09-21T10:28:54.148432+0000","last_fullsized":"2024-09-21T10:28:54.148432+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:33:28.795713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"413'15946","reported_seq":16034,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.149155+0000","last_change":"2024-09-21T10:24:25.830908+0000","last_active":"2024-09-21T10:28:54.149155+0000","last_peered":"2024-09-21T10:28:54.149155+0000","last_clean":"2024-09-21T10:28:54.149155+0000","last_became_active":"2024-09-21T10:24:25.830171+0000","last_became_peered":"2024-09-21T10:24:25.830171+0000","last_unstale":"2024-09-21T10:28:54.149155+0000","last_undegraded":"2024-09-21T10:28:54.149155+0000","last_fullsized":"2024-09-21T10:28:54.149155+0000","mapping_epoch":378,"log_start":"395'9100","ondisk_log_start":"395'9100","created":378,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:24:24.812572+0000","last_clean_scrub_stamp":"2024-09-21T10:24:24.812572+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:00.292732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15946,"num_write_kb":510272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":725,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.149150+0000","last_change":"2024-09-21T10:24:26.926717+0000","last_active":"2024-09-21T10:28:54.149150+0000","last_peered":"2024-09-21T10:28:54.149150+0000","last_clean":"2024-09-21T10:28:54.149150+0000","last_became_active":"2024-09-21T10:24:24.828205+0000","last_became_peered":"2024-09-21T10:24:24.828205+0000","last_unstale":"2024-09-21T10:28:54.149150+0000","last_undegraded":"2024-09-21T10:28:54.149150+0000","last_fullsized":"2024-09-21T10:28:54.149150+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:26.926661+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.926661+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:47.946104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":103,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108589+0000","last_change":"2024-09-21T10:25:48.762565+0000","last_active":"2024-09-21T10:28:48.108589+0000","last_peered":"2024-09-21T10:28:48.108589+0000","last_clean":"2024-09-21T10:28:48.108589+0000","last_became_active":"2024-09-21T10:25:48.761324+0000","last_became_peered":"2024-09-21T10:25:48.761324+0000","last_unstale":"2024-09-21T10:28:48.108589+0000","last_undegraded":"2024-09-21T10:28:48.108589+0000","last_fullsized":"2024-09-21T10:28:48.108589+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:58.885790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":727,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.598352+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:28:48.598352+0000","last_peered":"2024-09-21T10:28:48.598352+0000","last_clean":"2024-09-21T10:28:48.598352+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:28:48.598352+0000","last_undegraded":"2024-09-21T10:28:48.598352+0000","last_fullsized":"2024-09-21T10:28:48.598352+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":389,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.102198+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:28:48.102198+0000","last_peered":"2024-09-21T10:28:48.102198+0000","last_clean":"2024-09-21T10:28:48.102198+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:28:48.102198+0000","last_undegraded":"2024-09-21T10:28:48.102198+0000","last_fullsized":"2024-09-21T10:28:48.102198+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":413,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.103170+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:28:48.103170+0000","last_peered":"2024-09-21T10:28:48.103170+0000","last_clean":"2024-09-21T10:28:48.103170+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:28:48.103170+0000","last_undegraded":"2024-09-21T10:28:48.103170+0000","last_fullsized":"2024-09-21T10:28:48.103170+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":122,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.102806+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:28:48.102806+0000","last_peered":"2024-09-21T10:28:48.102806+0000","last_clean":"2024-09-21T10:28:48.102806+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:28:48.102806+0000","last_undegraded":"2024-09-21T10:28:48.102806+0000","last_fullsized":"2024-09-21T10:28:48.102806+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":117,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.144970+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:28:53.144970+0000","last_peered":"2024-09-21T10:28:53.144970+0000","last_clean":"2024-09-21T10:28:53.144970+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:28:53.144970+0000","last_undegraded":"2024-09-21T10:28:53.144970+0000","last_fullsized":"2024-09-21T10:28:53.144970+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":97,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.700171+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:28:53.700171+0000","last_peered":"2024-09-21T10:28:53.700171+0000","last_clean":"2024-09-21T10:28:53.700171+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:28:53.700171+0000","last_undegraded":"2024-09-21T10:28:53.700171+0000","last_fullsized":"2024-09-21T10:28:53.700171+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":125,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.145559+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:28:53.145559+0000","last_peered":"2024-09-21T10:28:53.145559+0000","last_clean":"2024-09-21T10:28:53.145559+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:28:53.145559+0000","last_undegraded":"2024-09-21T10:28:53.145559+0000","last_fullsized":"2024-09-21T10:28:53.145559+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":664,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148521+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:28:54.148521+0000","last_peered":"2024-09-21T10:28:54.148521+0000","last_clean":"2024-09-21T10:28:54.148521+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:28:54.148521+0000","last_undegraded":"2024-09-21T10:28:54.148521+0000","last_fullsized":"2024-09-21T10:28:54.148521+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":280,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.109019+0000","last_change":"2024-09-21T10:25:48.765786+0000","last_active":"2024-09-21T10:28:48.109019+0000","last_peered":"2024-09-21T10:28:48.109019+0000","last_clean":"2024-09-21T10:28:48.109019+0000","last_became_active":"2024-09-21T10:25:48.764011+0000","last_became_peered":"2024-09-21T10:25:48.764011+0000","last_unstale":"2024-09-21T10:28:48.109019+0000","last_undegraded":"2024-09-21T10:28:48.109019+0000","last_fullsized":"2024-09-21T10:28:48.109019+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:13.352641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":724,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148367+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:28:54.148367+0000","last_peered":"2024-09-21T10:28:54.148367+0000","last_clean":"2024-09-21T10:28:54.148367+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:28:54.148367+0000","last_undegraded":"2024-09-21T10:28:54.148367+0000","last_fullsized":"2024-09-21T10:28:54.148367+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":431,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.598324+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:28:48.598324+0000","last_peered":"2024-09-21T10:28:48.598324+0000","last_clean":"2024-09-21T10:28:48.598324+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:28:48.598324+0000","last_undegraded":"2024-09-21T10:28:48.598324+0000","last_fullsized":"2024-09-21T10:28:48.598324+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":752,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148527+0000","last_change":"2024-09-21T10:25:48.755341+0000","last_active":"2024-09-21T10:28:54.148527+0000","last_peered":"2024-09-21T10:28:54.148527+0000","last_clean":"2024-09-21T10:28:54.148527+0000","last_became_active":"2024-09-21T10:25:48.754620+0000","last_became_peered":"2024-09-21T10:25:48.754620+0000","last_unstale":"2024-09-21T10:28:54.148527+0000","last_undegraded":"2024-09-21T10:28:54.148527+0000","last_fullsized":"2024-09-21T10:28:54.148527+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:57:21.296549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":313,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108398+0000","last_change":"2024-09-21T10:25:48.765683+0000","last_active":"2024-09-21T10:28:48.108398+0000","last_peered":"2024-09-21T10:28:48.108398+0000","last_clean":"2024-09-21T10:28:48.108398+0000","last_became_active":"2024-09-21T10:25:48.765281+0000","last_became_peered":"2024-09-21T10:25:48.765281+0000","last_unstale":"2024-09-21T10:28:48.108398+0000","last_undegraded":"2024-09-21T10:28:48.108398+0000","last_fullsized":"2024-09-21T10:28:48.108398+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:41.798683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":439,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.890760+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:28:54.890760+0000","last_peered":"2024-09-21T10:28:54.890760+0000","last_clean":"2024-09-21T10:28:54.890760+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:28:54.890760+0000","last_undegraded":"2024-09-21T10:28:54.890760+0000","last_fullsized":"2024-09-21T10:28:54.890760+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":731,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148859+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:28:54.148859+0000","last_peered":"2024-09-21T10:28:54.148859+0000","last_clean":"2024-09-21T10:28:54.148859+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:28:54.148859+0000","last_undegraded":"2024-09-21T10:28:54.148859+0000","last_fullsized":"2024-09-21T10:28:54.148859+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":420,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.102572+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:28:48.102572+0000","last_peered":"2024-09-21T10:28:48.102572+0000","last_clean":"2024-09-21T10:28:48.102572+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:28:48.102572+0000","last_undegraded":"2024-09-21T10:28:48.102572+0000","last_fullsized":"2024-09-21T10:28:48.102572+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":728,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.591954+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:28:53.591954+0000","last_peered":"2024-09-21T10:28:53.591954+0000","last_clean":"2024-09-21T10:28:53.591954+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:28:53.591954+0000","last_undegraded":"2024-09-21T10:28:53.591954+0000","last_fullsized":"2024-09-21T10:28:53.591954+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":73,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108490+0000","last_change":"2024-09-21T10:25:48.767385+0000","last_active":"2024-09-21T10:28:48.108490+0000","last_peered":"2024-09-21T10:28:48.108490+0000","last_clean":"2024-09-21T10:28:48.108490+0000","last_became_active":"2024-09-21T10:25:48.764077+0000","last_became_peered":"2024-09-21T10:25:48.764077+0000","last_unstale":"2024-09-21T10:28:48.108490+0000","last_undegraded":"2024-09-21T10:28:48.108490+0000","last_fullsized":"2024-09-21T10:28:48.108490+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:06:51.969987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.102950+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:28:48.102950+0000","last_peered":"2024-09-21T10:28:48.102950+0000","last_clean":"2024-09-21T10:28:48.102950+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:28:48.102950+0000","last_undegraded":"2024-09-21T10:28:48.102950+0000","last_fullsized":"2024-09-21T10:28:48.102950+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":357,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.144979+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:28:53.144979+0000","last_peered":"2024-09-21T10:28:53.144979+0000","last_clean":"2024-09-21T10:28:53.144979+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:28:53.144979+0000","last_undegraded":"2024-09-21T10:28:53.144979+0000","last_fullsized":"2024-09-21T10:28:53.144979+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":127,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.102413+0000","last_change":"2024-09-21T10:25:48.753233+0000","last_active":"2024-09-21T10:28:48.102413+0000","last_peered":"2024-09-21T10:28:48.102413+0000","last_clean":"2024-09-21T10:28:48.102413+0000","last_became_active":"2024-09-21T10:25:48.752932+0000","last_became_peered":"2024-09-21T10:25:48.752932+0000","last_unstale":"2024-09-21T10:28:48.102413+0000","last_undegraded":"2024-09-21T10:28:48.102413+0000","last_fullsized":"2024-09-21T10:28:48.102413+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:49.456475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":335,"reported_epoch":415,"state":"active+clean","last_fresh":"2024-09-21T10:28:53.145519+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:28:53.145519+0000","last_peered":"2024-09-21T10:28:53.145519+0000","last_clean":"2024-09-21T10:28:53.145519+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:28:53.145519+0000","last_undegraded":"2024-09-21T10:28:53.145519+0000","last_fullsized":"2024-09-21T10:28:53.145519+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":755,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.148706+0000","last_change":"2024-09-21T10:25:48.755262+0000","last_active":"2024-09-21T10:28:54.148706+0000","last_peered":"2024-09-21T10:28:54.148706+0000","last_clean":"2024-09-21T10:28:54.148706+0000","last_became_active":"2024-09-21T10:25:48.754746+0000","last_became_peered":"2024-09-21T10:25:48.754746+0000","last_unstale":"2024-09-21T10:28:54.148706+0000","last_undegraded":"2024-09-21T10:28:54.148706+0000","last_fullsized":"2024-09-21T10:28:54.148706+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:28:17.312418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":706,"reported_epoch":416,"state":"active+clean","last_fresh":"2024-09-21T10:28:54.149693+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:28:54.149693+0000","last_peered":"2024-09-21T10:28:54.149693+0000","last_clean":"2024-09-21T10:28:54.149693+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:28:54.149693+0000","last_undegraded":"2024-09-21T10:28:54.149693+0000","last_fullsized":"2024-09-21T10:28:54.149693+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":112,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.103318+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:28:48.103318+0000","last_peered":"2024-09-21T10:28:48.103318+0000","last_clean":"2024-09-21T10:28:48.103318+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:28:48.103318+0000","last_undegraded":"2024-09-21T10:28:48.103318+0000","last_fullsized":"2024-09-21T10:28:48.103318+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":105,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-21T10:28:48.108284+0000","last_change":"2024-09-21T10:25:48.764108+0000","last_active":"2024-09-21T10:28:48.108284+0000","last_peered":"2024-09-21T10:28:48.108284+0000","last_clean":"2024-09-21T10:28:48.108284+0000","last_became_active":"2024-09-21T10:25:48.761361+0000","last_became_peered":"2024-09-21T10:25:48.761361+0000","last_unstale":"2024-09-21T10:28:48.108284+0000","last_undegraded":"2024-09-21T10:28:48.108284+0000","last_fullsized":"2024-09-21T10:28:48.108284+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:49.188529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8699,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":255600,"num_write_kb":8179137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9683,"num_bytes_recovered":391053312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125500,"ondisk_log_size":125500,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28818,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234471,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":861100,"kb_used_data":9632,"kb_used_omap":1,"kb_used_meta":851454,"kb_avail":92884052,"statfs":{"total":95995035648,"available":95113269248,"internally_reserved":0,"allocated":9863168,"data_stored":7750583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":871889355},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496109,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":704104,"kb_used_data":5212,"kb_used_omap":1,"kb_used_meta":698878,"kb_avail":93041048,"statfs":{"total":95995035648,"available":95274033152,"internally_reserved":0,"allocated":5337088,"data_stored":3228803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":715651527},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569404009,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":543088,"kb_used_data":5796,"kb_used_omap":7,"kb_used_meta":537272,"kb_avail":93202064,"statfs":{"total":95995035648,"available":95438913536,"internally_reserved":0,"allocated":5935104,"data_stored":3823352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":550167448},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019382,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":658156,"kb_used_data":5216,"kb_used_omap":1,"kb_used_meta":652926,"kb_avail":93086996,"statfs":{"total":95995035648,"available":95321083904,"internally_reserved":0,"allocated":5341184,"data_stored":3228860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":668596678},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435566,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1006492,"kb_used_data":5776,"kb_used_omap":5,"kb_used_meta":1000698,"kb_avail":92738660,"statfs":{"total":95995035648,"available":94964387840,"internally_reserved":0,"allocated":5914624,"data_stored":3814428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":1024714792},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599093,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":795164,"kb_used_data":5456,"kb_used_omap":7,"kb_used_meta":789688,"kb_avail":92949988,"statfs":{"total":95995035648,"available":95180787712,"internally_reserved":0,"allocated":5586944,"data_stored":3486966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":808640973},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762620,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":789280,"kb_used_data":9300,"kb_used_omap":7,"kb_used_meta":779960,"kb_avail":92955872,"statfs":{"total":95995035648,"available":95186812928,"internally_reserved":0,"allocated":9523200,"data_stored":7426375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":798679293},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":389,"seq":1670742278182,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":131916,"kb_used_data":9284,"kb_used_omap":12,"kb_used_meta":122611,"kb_avail":93613236,"statfs":{"total":95995035648,"available":95859953664,"internally_reserved":0,"allocated":9506816,"data_stored":7410502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12595,"internal_metadata":125554381},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11001,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:28:58.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:28:58.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:58.804 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:58.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:58.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:58.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:28:58.905 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:28:58.928 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:58.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:58.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:28:59.012 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:28:59.012 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:28:59.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:59.051 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:59.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:59.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:59.152 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:59.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:59.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:59.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:59.245 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:28:59.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:59.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:59.301 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:59.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:59.348 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:28:59 smithi032 ceph-mon[105748]: from='client.36171 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:28:59.348 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:28:59 smithi032 ceph-mon[101299]: from='client.36171 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:28:59.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:28:59.425 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:59.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:59.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:28:59.526 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:28:59.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:28:59 smithi045 ceph-mon[95976]: from='client.36171 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:28:59.549 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:59.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:59.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:28:59.650 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:28:59.667 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:28:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:28:59] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.51.0" 2024-09-21T10:28:59.674 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:28:59.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:59.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:28:59.776 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:28:59.799 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:59.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:59.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:28:59.814 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:28:59.900 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:28:59.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:28:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:28:59.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:00.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:00.047 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:00.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:00.091 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:00.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:00.170 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:00.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:00.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:00.272 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:00.295 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:00.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:00.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:00.397 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:00.419 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:00.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:00.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:00.521 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:00.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:00 smithi045 ceph-mon[95976]: pgmap v1309: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8699/84 objects degraded (10355.952%); 0 B/s, 261 objects/s recovering 2024-09-21T10:29:00.544 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:00.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:00.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:00.645 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:00.667 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:00.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:00.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:00.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:00 smithi032 ceph-mon[105748]: pgmap v1309: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8699/84 objects degraded (10355.952%); 0 B/s, 261 objects/s recovering 2024-09-21T10:29:00.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:00 smithi032 ceph-mon[101299]: pgmap v1309: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8699/84 objects degraded (10355.952%); 0 B/s, 261 objects/s recovering 2024-09-21T10:29:00.770 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:00.792 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:00.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:00.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:00.894 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:00.917 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:00.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:00.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:01.018 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:01.041 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:01.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:01.142 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:01.165 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:01.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:01.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:01.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:01.289 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:01.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:01.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:01.390 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:01.413 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:01.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:01.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:01.514 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:01.537 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:01.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:01.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:01.638 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:01.660 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:01.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:01.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:01.762 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:01.784 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:01.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:01.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:01.886 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:01.908 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:01.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:02.010 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:02.033 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:02.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:02.134 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:02.156 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:02.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:02.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:02.258 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:02.280 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:02.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:02.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:02.381 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:02.403 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:02.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:02.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:02 smithi032 ceph-mon[105748]: pgmap v1310: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8699/84 objects degraded (10355.952%); 0 B/s, 261 objects/s recovering 2024-09-21T10:29:02.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:02 smithi032 ceph-mon[101299]: pgmap v1310: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8699/84 objects degraded (10355.952%); 0 B/s, 261 objects/s recovering 2024-09-21T10:29:02.505 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:02.528 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:02.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:02.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:02 smithi045 ceph-mon[95976]: pgmap v1310: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8699/84 objects degraded (10355.952%); 0 B/s, 261 objects/s recovering 2024-09-21T10:29:02.629 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:02.652 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:02.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:02.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:02.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:02.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:02.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:02.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:02.843 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:02.879 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:02.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:02.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:02.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:03.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:03.027 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:03.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:03.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:03.081 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:03.129 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:03.152 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:03.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:03.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:03.253 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:03.277 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:03.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:03.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:03.312 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:03.321 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:03 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 7973/84 objects degraded (9491.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:29:03.321 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:03 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 7973/84 objects degraded (9491.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:29:03.379 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:03.401 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:03.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:03.503 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:03.525 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:03.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:03.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:03.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:03 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 7973/84 objects degraded (9491.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:29:03.627 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:03.649 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:03.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:03.751 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:03.773 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:03.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:03.874 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:03.898 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:03.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:03.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:04.000 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:04.022 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:04.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:04.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:04.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:04.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:04.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:04.249 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:04.271 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_4' removed 2024-09-21T10:29:04.272 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:04.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:04.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:04.373 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:04.395 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:04.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:04.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:04 smithi032 ceph-mon[101299]: pgmap v1311: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7973/84 objects degraded (9491.667%); 0 B/s, 295 objects/s recovering 2024-09-21T10:29:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:04 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1962625674' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:29:04.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:04.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:04 smithi032 ceph-mon[105748]: pgmap v1311: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7973/84 objects degraded (9491.667%); 0 B/s, 295 objects/s recovering 2024-09-21T10:29:04.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:04 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1962625674' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:29:04.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:04.497 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:04.519 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:04.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:04.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:04.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:04 smithi045 ceph-mon[95976]: pgmap v1311: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7973/84 objects degraded (9491.667%); 0 B/s, 295 objects/s recovering 2024-09-21T10:29:04.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:04 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1962625674' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:29:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:04.620 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:04.653 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:04.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:04.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:04.755 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:04.777 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:04.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:04.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:04.879 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:04.901 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:04.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:04.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:05.002 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:05.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:05.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:05.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:05.126 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:05.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:05.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:05.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:05.250 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:05.273 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:05.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:05.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:05.374 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:05.397 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:05.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:05.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:05.498 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:05.520 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:05.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:05.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:05.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1962625674' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:29:05.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:05 smithi045 ceph-mon[95976]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T10:29:05.622 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:05.634 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1962625674' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:29:05.634 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:05 smithi032 ceph-mon[105748]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T10:29:05.634 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1962625674' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:29:05.634 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:05 smithi032 ceph-mon[101299]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T10:29:05.649 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:05.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:05.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:05.658 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:29:05.658 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:29:05.658 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-09-21T10:29:05.659 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_5 16 2024-09-21T10:29:05.750 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:05.778 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:05.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:05.879 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:05.901 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:05.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:06.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:06.028 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:06.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:06.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:06.130 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:06.156 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:06.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:06.257 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:06.280 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:06.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:06.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:06.382 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:06.404 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:06.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:06.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:06.506 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:06.525 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:29:06.525 INFO:teuthology.orchestra.run.smithi032.stdout:{"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1836,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":417,"num_osds":8,"num_up_osds":8,"osd_up_since":1726914347,"num_in_osds":8,"osd_in_since":1726913989,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83}],"num_pgs":83,"num_pools":4,"num_objects":28,"data_bytes":4790245,"bytes_used":5621153792,"bytes_avail":762339131392,"bytes_total":767960285184},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi032.fdolcm","status":"up:active","gid":14655}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.32:8443/","prometheus":"http://172.21.15.32:9283/"}},"servicemap":{"epoch":29,"modified":"2024-09-21T10:27:32.636453+0000","services":{}},"progress_events":{}} 2024-09-21T10:29:06.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:06 smithi032 ceph-mon[105748]: pgmap v1313: 83 pgs: 83 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:06.525 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:06 smithi032 ceph-mon[105748]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7973/84 objects degraded (9491.667%), 2 pgs degraded, 2 pgs undersized) 2024-09-21T10:29:06.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:06 smithi032 ceph-mon[101299]: pgmap v1313: 83 pgs: 83 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:06.526 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:06 smithi032 ceph-mon[101299]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7973/84 objects degraded (9491.667%), 2 pgs degraded, 2 pgs undersized) 2024-09-21T10:29:06.528 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:06.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:06.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:06 smithi045 ceph-mon[95976]: pgmap v1313: 83 pgs: 83 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:06.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:06 smithi045 ceph-mon[95976]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7973/84 objects degraded (9491.667%), 2 pgs degraded, 2 pgs undersized) 2024-09-21T10:29:06.629 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:06.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:06.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:06.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:06.757 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:06.780 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:06.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:06.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:06.882 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:06.907 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:06.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:06.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:07.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:07.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:07.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:07.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:07.133 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:07.160 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:07.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:07.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:07.262 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:07.288 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:07.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:07.389 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:07.414 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:07.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:07.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:07.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:07 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2424609210' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:29:07.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:07 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2424609210' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:29:07.516 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:07.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:07 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2424609210' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:29:07.541 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:07.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:07.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:07.643 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:07.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:07.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:07.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:07.768 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:07.795 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:07.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:07.896 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:07.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:07.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:08.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:08.048 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:08.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:08.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:08.149 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:08.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:08.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:08.230 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-09-21T10:29:08.230 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:29:08.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:08.297 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:08.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:08.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:08.399 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:08.421 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:08.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:08.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:08 smithi032 ceph-mon[105748]: pgmap v1314: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:08.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:08 smithi032 ceph-mon[101299]: pgmap v1314: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:08.523 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:08 smithi045 ceph-mon[95976]: pgmap v1314: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:08.550 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:08.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:08.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:08.651 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:08.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:08.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:08.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:08.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:08.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:08.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:08.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:08.925 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:08.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:08.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:09.027 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:09.054 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:09.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:09.155 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:09.179 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:09.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:09.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:09.281 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:09.308 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:09.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:09.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:09.409 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:09.431 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:09.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:09.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:09.533 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:09.559 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:09.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:09.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:09.661 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:09.684 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:09.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:09.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:09.786 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:09.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:09.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:09.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:09.905 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:29:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:29:09] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.51.0" 2024-09-21T10:29:09.914 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:09.941 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:09.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:10.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:10.068 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:10.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:10.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:10.170 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:10.196 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:10.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:10.288 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:29:10.298 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:10.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:10.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:10.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:10.425 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:10.448 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:10.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:10.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:10.550 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:10.560 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:10 smithi032 ceph-mon[105748]: pgmap v1315: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:10.560 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:10 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3249639488' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:29:10.560 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:10 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:29:10.561 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:10 smithi032 ceph-mon[101299]: pgmap v1315: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:10.561 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:10 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3249639488' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:29:10.561 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:10 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:29:10.576 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:10.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:10.678 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:10.701 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:10.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:10.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:10.784 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:10 smithi045 ceph-mon[95976]: pgmap v1315: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:10.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:10 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3249639488' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:29:10.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:10 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:29:10.802 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:10.824 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:10.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:10.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:10.952 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:10.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:10.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:11.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:11.077 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:11.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:11.178 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:11.202 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:11.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:11.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:11.304 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:11.330 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:11.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:11.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:11.431 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:11.454 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:11.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:11.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:11.556 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:11.578 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:11.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:11.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:11.680 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:11.706 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:11 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:29:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:11 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:29:11.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:11 smithi032 ceph-mon[101299]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T10:29:11.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:11 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:29:11.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:11 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:29:11.708 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:11 smithi032 ceph-mon[105748]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T10:29:11.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:11.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:11.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:11 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:29:11.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:11 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:29:11.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:11 smithi045 ceph-mon[95976]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T10:29:11.809 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:11.836 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:11.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:11.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:11.938 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:11.964 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:11.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:11.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:11.999 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:29:11.999 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:29:12.065 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:12.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:12.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:12.193 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:12.218 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:12.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:12.320 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:12.346 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:12.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:12.448 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:12.456 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:29:12.456 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:29:12.470 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:12.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:12.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:12.474 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:29:12.474 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:29:12.490 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:29:12.491 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:29:12.506 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:29:12.506 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:29:12.524 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:29:12.524 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:29:12.541 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:29:12.541 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:29:12.558 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:29:12.558 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:29:12.558 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:12 smithi032 ceph-mon[105748]: pgmap v1317: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:12.558 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:12 smithi032 ceph-mon[101299]: pgmap v1317: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:12.572 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:12.574 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:29:12.574 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:29:12.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:12.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:12.699 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:12.721 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:12.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:12.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:12.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:12 smithi045 ceph-mon[95976]: pgmap v1317: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:12.823 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:12.849 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:12.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:12.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:12.951 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:12.976 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:12.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:13.078 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:13.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:13.101 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:13.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:13.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:13.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:13.225 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:13.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:13.326 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:13.352 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:13.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:13.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:13.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/448093750' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:13.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/448093750' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:13.454 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:13.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:13.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:13.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:13.580 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:13.602 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:13.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:13.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:13.704 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:13.731 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:13.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:13.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/448093750' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:13.786 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:29:13.832 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:13.858 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:13.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:13.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:13.885 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:13.890 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:13.961 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:13.983 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:13.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:13.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:14.085 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:14.112 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:14.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:14.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:14.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:14.240 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:14.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:14.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:14.341 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:14.364 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:14.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:14.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:14.401 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:14 smithi032 ceph-mon[105748]: pgmap v1318: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:14.401 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:14 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:14.401 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:14 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:14.401 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:14 smithi032 ceph-mon[101299]: pgmap v1318: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:14.401 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:14 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:14.402 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:14 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:14.402 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:29:14.466 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:14.493 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:14.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:14.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:14.594 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:14.621 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:14.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:14.723 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:14.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:14.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:14.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:14.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:14 smithi045 ceph-mon[95976]: pgmap v1318: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:14.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:14 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:14.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:14 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:14.851 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:14.878 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:14.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:14.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:14.979 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:15.006 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:15.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:15.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:15.107 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:15.134 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:15.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:15.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:15.235 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:15.259 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:15.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:15.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:15.360 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:15.386 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:15.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:15.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:15.468 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:29:15.487 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:15.532 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:15.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:15.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:15.633 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:15.660 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:15.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:15.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:15.761 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:15.784 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:15.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:15.886 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:15.912 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:15.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:15.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:16.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:16.037 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:16.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:16.139 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:16.161 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:16.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:16.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:16.263 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:16.286 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:16.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:16.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:16.387 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:16.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:16.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:16.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:16.512 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:16.539 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:16.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:16.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:16.604 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:16 smithi032 ceph-mon[105748]: pgmap v1319: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:16.604 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:16 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:29:16.604 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:16 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:29:16.604 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:16 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:16.604 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:16 smithi032 ceph-mon[101299]: pgmap v1319: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:16.605 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:16 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:29:16.605 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:16 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:29:16.605 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:16 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:16.641 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:16.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:16.670 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:16.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:16.771 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:16.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:16 smithi045 ceph-mon[95976]: pgmap v1319: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:16.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:16 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:29:16.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:16 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:29:16.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:16 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:16.799 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:16.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:16.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:16.901 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:16.927 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:16.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:16.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:17.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:17.051 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:17.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:17.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:17.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:17.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:17.182 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:17.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:17.283 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:17.308 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:17.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:17.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:17.310 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_5' created 2024-09-21T10:29:17.410 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:17.433 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:17.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:17.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:17.534 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:17.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:17.557 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:17.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:17.580 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:17.600 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:17.658 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:17.680 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:17.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:17.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:17.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:17 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1023906175' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:29:17.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:17 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1023906175' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:29:17.782 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:17.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:17 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1023906175' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:29:17.805 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:17.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:17.907 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:17.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:17.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:17.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:18.035 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:18.043 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-09-21T10:29:18.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:18.059 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:18.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:18.160 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:18.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:18.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:18.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:18.288 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:18.311 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:18.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:18.414 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:18.437 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:18.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:18.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:18.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:18.561 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:18.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:18.663 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:18.689 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:18.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:18.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:18.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:18 smithi032 ceph-mon[101299]: pgmap v1320: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:18.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:18 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1023906175' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-21T10:29:18.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:18 smithi032 ceph-mon[101299]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T10:29:18.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:18 smithi032 ceph-mon[105748]: pgmap v1320: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:18.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:18 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1023906175' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-21T10:29:18.707 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:18 smithi032 ceph-mon[105748]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T10:29:18.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:18 smithi045 ceph-mon[95976]: pgmap v1320: 83 pgs: 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:18.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:18 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1023906175' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-21T10:29:18.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:18 smithi045 ceph-mon[95976]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T10:29:18.791 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:18.818 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:18.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:18.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:18.919 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:18.946 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:18.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:19.048 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:19.074 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:19.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:19.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:19.202 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:19.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:19.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:19.304 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:19.332 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:19.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:19.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:19.456 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:19.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:19.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:19.535 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:19 smithi032 ceph-mon[101299]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T10:29:19.535 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:19.535 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:19.536 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:19 smithi032 ceph-mon[105748]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T10:29:19.536 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:19.536 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:19.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:19.583 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:19.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:19.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:19.685 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:19.707 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:19.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:19.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:19.716 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:29:19.717 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:29:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:19 smithi045 ceph-mon[95976]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T10:29:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:19.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:19.809 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:19.822 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:29:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:29:19] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.51.0" 2024-09-21T10:29:19.836 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:19.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:19.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:19.937 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:19.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:19.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:20.062 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:20.084 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:20.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:20.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:20.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:20.207 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:20.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:20.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:20.309 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:20.332 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:20.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:20.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:20.433 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:20.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:20.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:20.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:20 smithi032 ceph-mon[101299]: pgmap v1323: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:20 smithi032 ceph-mon[101299]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T10:29:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:20 smithi032 ceph-mon[101299]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:29:20.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:20 smithi032 ceph-mon[101299]: from='client.46121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:29:20.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:20 smithi032 ceph-mon[105748]: pgmap v1323: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:20.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:20 smithi032 ceph-mon[105748]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T10:29:20.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:20 smithi032 ceph-mon[105748]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:29:20.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:20 smithi032 ceph-mon[105748]: from='client.46121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:29:20.557 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:20.581 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:20.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:20.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:20.682 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:20.685 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:20 smithi045 ceph-mon[95976]: pgmap v1323: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:20.686 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:20 smithi045 ceph-mon[95976]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T10:29:20.686 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:20 smithi045 ceph-mon[95976]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:29:20.686 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:20 smithi045 ceph-mon[95976]: from='client.46121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:29:20.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:20.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:20.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:20.811 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:20.837 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:20.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:20.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:20.961 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:20.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:20.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:21.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:21.086 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:21.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:21.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:21.188 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:21.210 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:21.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:21.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:21.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:21.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:21.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:21.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:21.440 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:21.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:21.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:21.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:21.569 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:21.591 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:21.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:21.693 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:21.706 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:21 smithi032 ceph-mon[105748]: pgmap v1325: 99 pgs: 5 unknown, 94 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:21.707 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:21 smithi032 ceph-mon[101299]: pgmap v1325: 99 pgs: 5 unknown, 94 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:21.715 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:21.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:21.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:21 smithi045 ceph-mon[95976]: pgmap v1325: 99 pgs: 5 unknown, 94 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:21.816 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:21.843 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:21.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:21.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:21.945 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:21.971 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:21.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:21.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:22.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:22.092 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:22.099 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:22.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:22.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:22.201 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:22.227 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:22.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:22.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:22.329 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:22.344 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:22.355 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:22.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:22.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:22.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:22.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:22.480 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:22.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:22.563 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:29:22.581 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:22.602 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:22 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3128720355' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:29:22.602 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:22 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:29:22.602 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:22 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3128720355' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:29:22.602 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:22 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:29:22.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:22.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:22.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:22.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:22.734 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:22.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:22.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:22.836 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:22.858 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:22.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:22.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:22.982 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:22.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:22.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:22 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3128720355' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:29:23.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:22 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:29:23.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:23.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:23.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:23.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:23.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:23.231 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1324,"stamp":"2024-09-21T10:29:19.316121+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":908,"num_write_kb":12625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":285,"num_bytes_recovered":16782981,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2674,"ondisk_log_size":2674,"up":276,"acting":276,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5513308,"kb_used_data":56052,"kb_used_omap":45,"kb_used_meta":5457106,"kb_avail":744447908,"statfs":{"total":767960285184,"available":762314657792,"internally_reserved":0,"allocated":57397248,"data_stored":40409419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46417,"internal_metadata":5588077231},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.632057"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":715,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.286119+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:29:10.286119+0000","last_peered":"2024-09-21T10:29:10.286119+0000","last_clean":"2024-09-21T10:29:10.286119+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:29:10.286119+0000","last_undegraded":"2024-09-21T10:29:10.286119+0000","last_fullsized":"2024-09-21T10:29:10.286119+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":425,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312445+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:29:17.312445+0000","last_peered":"2024-09-21T10:29:17.312445+0000","last_clean":"2024-09-21T10:29:17.312445+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:29:17.312445+0000","last_undegraded":"2024-09-21T10:29:17.312445+0000","last_fullsized":"2024-09-21T10:29:17.312445+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.276144+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:29:04.276144+0000","last_peered":"2024-09-21T10:29:04.276144+0000","last_clean":"2024-09-21T10:29:04.276144+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:29:04.276144+0000","last_undegraded":"2024-09-21T10:29:04.276144+0000","last_fullsized":"2024-09-21T10:29:04.276144+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":82,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603702+0000","last_change":"2024-09-21T10:25:48.765404+0000","last_active":"2024-09-21T10:29:10.603702+0000","last_peered":"2024-09-21T10:29:10.603702+0000","last_clean":"2024-09-21T10:29:10.603702+0000","last_became_active":"2024-09-21T10:25:48.764230+0000","last_became_peered":"2024-09-21T10:25:48.764230+0000","last_unstale":"2024-09-21T10:29:10.603702+0000","last_undegraded":"2024-09-21T10:29:10.603702+0000","last_fullsized":"2024-09-21T10:29:10.603702+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:38.551941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":83,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603900+0000","last_change":"2024-09-21T10:25:48.764391+0000","last_active":"2024-09-21T10:29:10.603900+0000","last_peered":"2024-09-21T10:29:10.603900+0000","last_clean":"2024-09-21T10:29:10.603900+0000","last_became_active":"2024-09-21T10:25:48.763375+0000","last_became_peered":"2024-09-21T10:25:48.763375+0000","last_unstale":"2024-09-21T10:29:10.603900+0000","last_undegraded":"2024-09-21T10:29:10.603900+0000","last_fullsized":"2024-09-21T10:29:10.603900+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:03.457716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":396,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311991+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:29:17.311991+0000","last_peered":"2024-09-21T10:29:17.311991+0000","last_clean":"2024-09-21T10:29:17.311991+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:29:17.311991+0000","last_undegraded":"2024-09-21T10:29:17.311991+0000","last_fullsized":"2024-09-21T10:29:17.311991+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":736,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.286184+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:29:10.286184+0000","last_peered":"2024-09-21T10:29:10.286184+0000","last_clean":"2024-09-21T10:29:10.286184+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:29:10.286184+0000","last_undegraded":"2024-09-21T10:29:10.286184+0000","last_fullsized":"2024-09-21T10:29:10.286184+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":492,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.316663+0000","last_change":"2024-09-21T10:25:48.752712+0000","last_active":"2024-09-21T10:29:17.316663+0000","last_peered":"2024-09-21T10:29:17.316663+0000","last_clean":"2024-09-21T10:29:17.316663+0000","last_became_active":"2024-09-21T10:25:48.752204+0000","last_became_peered":"2024-09-21T10:25:48.752204+0000","last_unstale":"2024-09-21T10:29:17.316663+0000","last_undegraded":"2024-09-21T10:29:17.316663+0000","last_fullsized":"2024-09-21T10:29:17.316663+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:40.068520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-21T10:29:17.303121+0000","last_change":"2024-09-21T10:29:17.303121+0000","last_active":"2024-09-21T10:29:17.303121+0000","last_peered":"2024-09-21T10:29:17.303121+0000","last_clean":"2024-09-21T10:29:17.303121+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-21T10:29:17.303121+0000","last_undegraded":"2024-09-21T10:29:17.303121+0000","last_fullsized":"2024-09-21T10:29:17.303121+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":127,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312440+0000","last_change":"2024-09-21T10:25:48.755730+0000","last_active":"2024-09-21T10:29:17.312440+0000","last_peered":"2024-09-21T10:29:17.312440+0000","last_clean":"2024-09-21T10:29:17.312440+0000","last_became_active":"2024-09-21T10:25:48.755344+0000","last_became_peered":"2024-09-21T10:25:48.755344+0000","last_unstale":"2024-09-21T10:29:17.312440+0000","last_undegraded":"2024-09-21T10:29:17.312440+0000","last_fullsized":"2024-09-21T10:29:17.312440+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:37.725090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":128,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.271279+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:29:04.271279+0000","last_peered":"2024-09-21T10:29:04.271279+0000","last_clean":"2024-09-21T10:29:04.271279+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:29:04.271279+0000","last_undegraded":"2024-09-21T10:29:04.271279+0000","last_fullsized":"2024-09-21T10:29:04.271279+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"0'0","reported_seq":10,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-21T10:29:18.322126+0000","last_change":"2024-09-21T10:29:18.320824+0000","last_active":"2024-09-21T10:29:18.322126+0000","last_peered":"2024-09-21T10:29:18.322126+0000","last_clean":"2024-09-21T10:29:18.322126+0000","last_became_active":"2024-09-21T10:29:18.320520+0000","last_became_peered":"2024-09-21T10:29:18.320520+0000","last_unstale":"2024-09-21T10:29:18.322126+0000","last_undegraded":"2024-09-21T10:29:18.322126+0000","last_fullsized":"2024-09-21T10:29:18.322126+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:23.749267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":99,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312718+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:29:17.312718+0000","last_peered":"2024-09-21T10:29:17.312718+0000","last_clean":"2024-09-21T10:29:17.312718+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:29:17.312718+0000","last_undegraded":"2024-09-21T10:29:17.312718+0000","last_fullsized":"2024-09-21T10:29:17.312718+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":118,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311316+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:29:17.311316+0000","last_peered":"2024-09-21T10:29:17.311316+0000","last_clean":"2024-09-21T10:29:17.311316+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:29:17.311316+0000","last_undegraded":"2024-09-21T10:29:17.311316+0000","last_fullsized":"2024-09-21T10:29:17.311316+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-21T10:29:17.303121+0000","last_change":"2024-09-21T10:29:17.303121+0000","last_active":"2024-09-21T10:29:17.303121+0000","last_peered":"2024-09-21T10:29:17.303121+0000","last_clean":"2024-09-21T10:29:17.303121+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-21T10:29:17.303121+0000","last_undegraded":"2024-09-21T10:29:17.303121+0000","last_fullsized":"2024-09-21T10:29:17.303121+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":421,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312948+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:29:17.312948+0000","last_peered":"2024-09-21T10:29:17.312948+0000","last_clean":"2024-09-21T10:29:17.312948+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:29:17.312948+0000","last_undegraded":"2024-09-21T10:29:17.312948+0000","last_fullsized":"2024-09-21T10:29:17.312948+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":440,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.316653+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:29:17.316653+0000","last_peered":"2024-09-21T10:29:17.316653+0000","last_clean":"2024-09-21T10:29:17.316653+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:29:17.316653+0000","last_undegraded":"2024-09-21T10:29:17.316653+0000","last_fullsized":"2024-09-21T10:29:17.316653+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-21T10:29:17.303121+0000","last_change":"2024-09-21T10:29:17.303121+0000","last_active":"2024-09-21T10:29:17.303121+0000","last_peered":"2024-09-21T10:29:17.303121+0000","last_clean":"2024-09-21T10:29:17.303121+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-21T10:29:17.303121+0000","last_undegraded":"2024-09-21T10:29:17.303121+0000","last_fullsized":"2024-09-21T10:29:17.303121+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":353,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.286451+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:29:10.286451+0000","last_peered":"2024-09-21T10:29:10.286451+0000","last_clean":"2024-09-21T10:29:10.286451+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:29:10.286451+0000","last_undegraded":"2024-09-21T10:29:10.286451+0000","last_fullsized":"2024-09-21T10:29:10.286451+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":273,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.604033+0000","last_change":"2024-09-21T10:25:48.763096+0000","last_active":"2024-09-21T10:29:10.604033+0000","last_peered":"2024-09-21T10:29:10.604033+0000","last_clean":"2024-09-21T10:29:10.604033+0000","last_became_active":"2024-09-21T10:25:48.761327+0000","last_became_peered":"2024-09-21T10:25:48.761327+0000","last_unstale":"2024-09-21T10:29:10.604033+0000","last_undegraded":"2024-09-21T10:29:10.604033+0000","last_fullsized":"2024-09-21T10:29:10.604033+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:23.154333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-21T10:29:17.303121+0000","last_change":"2024-09-21T10:29:17.303121+0000","last_active":"2024-09-21T10:29:17.303121+0000","last_peered":"2024-09-21T10:29:17.303121+0000","last_clean":"2024-09-21T10:29:17.303121+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-21T10:29:17.303121+0000","last_undegraded":"2024-09-21T10:29:17.303121+0000","last_fullsized":"2024-09-21T10:29:17.303121+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":706,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311769+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:29:17.311769+0000","last_peered":"2024-09-21T10:29:17.311769+0000","last_clean":"2024-09-21T10:29:17.311769+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:29:17.311769+0000","last_undegraded":"2024-09-21T10:29:17.311769+0000","last_fullsized":"2024-09-21T10:29:17.311769+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":777,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285871+0000","last_change":"2024-09-21T10:25:48.755999+0000","last_active":"2024-09-21T10:29:10.285871+0000","last_peered":"2024-09-21T10:29:10.285871+0000","last_clean":"2024-09-21T10:29:10.285871+0000","last_became_active":"2024-09-21T10:25:48.754257+0000","last_became_peered":"2024-09-21T10:25:48.754257+0000","last_unstale":"2024-09-21T10:29:10.285871+0000","last_undegraded":"2024-09-21T10:29:10.285871+0000","last_fullsized":"2024-09-21T10:29:10.285871+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:06.025116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":118,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311959+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:29:17.311959+0000","last_peered":"2024-09-21T10:29:17.311959+0000","last_clean":"2024-09-21T10:29:17.311959+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:29:17.311959+0000","last_undegraded":"2024-09-21T10:29:17.311959+0000","last_fullsized":"2024-09-21T10:29:17.311959+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-21T10:29:17.303121+0000","last_change":"2024-09-21T10:29:17.303121+0000","last_active":"2024-09-21T10:29:17.303121+0000","last_peered":"2024-09-21T10:29:17.303121+0000","last_clean":"2024-09-21T10:29:17.303121+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-21T10:29:17.303121+0000","last_undegraded":"2024-09-21T10:29:17.303121+0000","last_fullsized":"2024-09-21T10:29:17.303121+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":477,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.271199+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:29:04.271199+0000","last_peered":"2024-09-21T10:29:04.271199+0000","last_clean":"2024-09-21T10:29:04.271199+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:29:04.271199+0000","last_undegraded":"2024-09-21T10:29:04.271199+0000","last_fullsized":"2024-09-21T10:29:04.271199+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":430,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.316715+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:29:17.316715+0000","last_peered":"2024-09-21T10:29:17.316715+0000","last_clean":"2024-09-21T10:29:17.316715+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:29:17.316715+0000","last_undegraded":"2024-09-21T10:29:17.316715+0000","last_fullsized":"2024-09-21T10:29:17.316715+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":118,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312627+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:29:17.312627+0000","last_peered":"2024-09-21T10:29:17.312627+0000","last_clean":"2024-09-21T10:29:17.312627+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:29:17.312627+0000","last_undegraded":"2024-09-21T10:29:17.312627+0000","last_fullsized":"2024-09-21T10:29:17.312627+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":10,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-21T10:29:18.323417+0000","last_change":"2024-09-21T10:29:18.322112+0000","last_active":"2024-09-21T10:29:18.323417+0000","last_peered":"2024-09-21T10:29:18.323417+0000","last_clean":"2024-09-21T10:29:18.323417+0000","last_became_active":"2024-09-21T10:29:18.321784+0000","last_became_peered":"2024-09-21T10:29:18.321784+0000","last_unstale":"2024-09-21T10:29:18.323417+0000","last_undegraded":"2024-09-21T10:29:18.323417+0000","last_fullsized":"2024-09-21T10:29:18.323417+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:10.300972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":118,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311153+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:29:17.311153+0000","last_peered":"2024-09-21T10:29:17.311153+0000","last_clean":"2024-09-21T10:29:17.311153+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:29:17.311153+0000","last_undegraded":"2024-09-21T10:29:17.311153+0000","last_fullsized":"2024-09-21T10:29:17.311153+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":118,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312393+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:29:17.312393+0000","last_peered":"2024-09-21T10:29:17.312393+0000","last_clean":"2024-09-21T10:29:17.312393+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:29:17.312393+0000","last_undegraded":"2024-09-21T10:29:17.312393+0000","last_fullsized":"2024-09-21T10:29:17.312393+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":10,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-21T10:29:18.321028+0000","last_change":"2024-09-21T10:29:18.319916+0000","last_active":"2024-09-21T10:29:18.321028+0000","last_peered":"2024-09-21T10:29:18.321028+0000","last_clean":"2024-09-21T10:29:18.321028+0000","last_became_active":"2024-09-21T10:29:18.319395+0000","last_became_peered":"2024-09-21T10:29:18.319395+0000","last_unstale":"2024-09-21T10:29:18.321028+0000","last_undegraded":"2024-09-21T10:29:18.321028+0000","last_fullsized":"2024-09-21T10:29:18.321028+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:46.166177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":655,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.271457+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:29:04.271457+0000","last_peered":"2024-09-21T10:29:04.271457+0000","last_clean":"2024-09-21T10:29:04.271457+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:29:04.271457+0000","last_undegraded":"2024-09-21T10:29:04.271457+0000","last_fullsized":"2024-09-21T10:29:04.271457+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":744,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285536+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:29:10.285536+0000","last_peered":"2024-09-21T10:29:10.285536+0000","last_clean":"2024-09-21T10:29:10.285536+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:29:10.285536+0000","last_undegraded":"2024-09-21T10:29:10.285536+0000","last_fullsized":"2024-09-21T10:29:10.285536+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":437,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.700377+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:29:10.700377+0000","last_peered":"2024-09-21T10:29:10.700377+0000","last_clean":"2024-09-21T10:29:10.700377+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:29:10.700377+0000","last_undegraded":"2024-09-21T10:29:10.700377+0000","last_fullsized":"2024-09-21T10:29:10.700377+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"0'0","reported_seq":10,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-21T10:29:18.322529+0000","last_change":"2024-09-21T10:29:18.321006+0000","last_active":"2024-09-21T10:29:18.322529+0000","last_peered":"2024-09-21T10:29:18.322529+0000","last_clean":"2024-09-21T10:29:18.322529+0000","last_became_active":"2024-09-21T10:29:18.320643+0000","last_became_peered":"2024-09-21T10:29:18.320643+0000","last_unstale":"2024-09-21T10:29:18.322529+0000","last_undegraded":"2024-09-21T10:29:18.322529+0000","last_fullsized":"2024-09-21T10:29:18.322529+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:44.782234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":705,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311516+0000","last_change":"2024-09-21T10:25:48.754725+0000","last_active":"2024-09-21T10:29:17.311516+0000","last_peered":"2024-09-21T10:29:17.311516+0000","last_clean":"2024-09-21T10:29:17.311516+0000","last_became_active":"2024-09-21T10:25:48.754341+0000","last_became_peered":"2024-09-21T10:25:48.754341+0000","last_unstale":"2024-09-21T10:29:17.311516+0000","last_undegraded":"2024-09-21T10:29:17.311516+0000","last_fullsized":"2024-09-21T10:29:17.311516+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:25.191555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:25.191555+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:08.981054+0000","scrub_duration":0.0015044590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":732,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.271799+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:29:04.271799+0000","last_peered":"2024-09-21T10:29:04.271799+0000","last_clean":"2024-09-21T10:29:04.271799+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:29:04.271799+0000","last_undegraded":"2024-09-21T10:29:04.271799+0000","last_fullsized":"2024-09-21T10:29:04.271799+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":676,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285612+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:29:10.285612+0000","last_peered":"2024-09-21T10:29:10.285612+0000","last_clean":"2024-09-21T10:29:10.285612+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:29:10.285612+0000","last_undegraded":"2024-09-21T10:29:10.285612+0000","last_fullsized":"2024-09-21T10:29:10.285612+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":397,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.317161+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:29:17.317161+0000","last_peered":"2024-09-21T10:29:17.317161+0000","last_clean":"2024-09-21T10:29:17.317161+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:29:17.317161+0000","last_undegraded":"2024-09-21T10:29:17.317161+0000","last_fullsized":"2024-09-21T10:29:17.317161+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-21T10:29:17.303121+0000","last_change":"2024-09-21T10:29:17.303121+0000","last_active":"2024-09-21T10:29:17.303121+0000","last_peered":"2024-09-21T10:29:17.303121+0000","last_clean":"2024-09-21T10:29:17.303121+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-21T10:29:17.303121+0000","last_undegraded":"2024-09-21T10:29:17.303121+0000","last_fullsized":"2024-09-21T10:29:17.303121+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":693,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.700070+0000","last_change":"2024-09-21T10:24:27.359111+0000","last_active":"2024-09-21T10:29:10.700070+0000","last_peered":"2024-09-21T10:29:10.700070+0000","last_clean":"2024-09-21T10:29:10.700070+0000","last_became_active":"2024-09-21T10:24:24.825850+0000","last_became_peered":"2024-09-21T10:24:24.825850+0000","last_unstale":"2024-09-21T10:29:10.700070+0000","last_undegraded":"2024-09-21T10:29:10.700070+0000","last_fullsized":"2024-09-21T10:29:10.700070+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:27.359065+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:24:27.359065+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:06.871319+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1077,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311659+0000","last_change":"2024-09-21T10:25:48.755667+0000","last_active":"2024-09-21T10:29:17.311659+0000","last_peered":"2024-09-21T10:29:17.311659+0000","last_clean":"2024-09-21T10:29:17.311659+0000","last_became_active":"2024-09-21T10:25:48.755060+0000","last_became_peered":"2024-09-21T10:25:48.755060+0000","last_unstale":"2024-09-21T10:29:17.311659+0000","last_undegraded":"2024-09-21T10:29:17.311659+0000","last_fullsized":"2024-09-21T10:29:17.311659+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:09.230030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":275,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603544+0000","last_change":"2024-09-21T10:25:48.754659+0000","last_active":"2024-09-21T10:29:10.603544+0000","last_peered":"2024-09-21T10:29:10.603544+0000","last_clean":"2024-09-21T10:29:10.603544+0000","last_became_active":"2024-09-21T10:25:48.754230+0000","last_became_peered":"2024-09-21T10:25:48.754230+0000","last_unstale":"2024-09-21T10:29:10.603544+0000","last_undegraded":"2024-09-21T10:29:10.603544+0000","last_fullsized":"2024-09-21T10:29:10.603544+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:12.555868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":82,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603721+0000","last_change":"2024-09-21T10:25:48.761249+0000","last_active":"2024-09-21T10:29:10.603721+0000","last_peered":"2024-09-21T10:29:10.603721+0000","last_clean":"2024-09-21T10:29:10.603721+0000","last_became_active":"2024-09-21T10:25:48.759359+0000","last_became_peered":"2024-09-21T10:25:48.759359+0000","last_unstale":"2024-09-21T10:29:10.603721+0000","last_undegraded":"2024-09-21T10:29:10.603721+0000","last_fullsized":"2024-09-21T10:29:10.603721+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:33.510936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-09-21T10:29:17.303121+0000","last_change":"2024-09-21T10:29:17.303121+0000","last_active":"2024-09-21T10:29:17.303121+0000","last_peered":"2024-09-21T10:29:17.303121+0000","last_clean":"2024-09-21T10:29:17.303121+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-21T10:29:17.303121+0000","last_undegraded":"2024-09-21T10:29:17.303121+0000","last_fullsized":"2024-09-21T10:29:17.303121+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":721,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312848+0000","last_change":"2024-09-21T10:25:48.755509+0000","last_active":"2024-09-21T10:29:17.312848+0000","last_peered":"2024-09-21T10:29:17.312848+0000","last_clean":"2024-09-21T10:29:17.312848+0000","last_became_active":"2024-09-21T10:25:48.754558+0000","last_became_peered":"2024-09-21T10:25:48.754558+0000","last_unstale":"2024-09-21T10:29:17.312848+0000","last_undegraded":"2024-09-21T10:29:17.312848+0000","last_fullsized":"2024-09-21T10:29:17.312848+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:11.262503+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1276,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.668053+0000","last_change":"2024-09-21T10:25:48.753359+0000","last_active":"2024-09-21T10:29:10.668053+0000","last_peered":"2024-09-21T10:29:10.668053+0000","last_clean":"2024-09-21T10:29:10.668053+0000","last_became_active":"2024-09-21T10:25:48.752653+0000","last_became_peered":"2024-09-21T10:25:48.752653+0000","last_unstale":"2024-09-21T10:29:10.668053+0000","last_undegraded":"2024-09-21T10:29:10.668053+0000","last_fullsized":"2024-09-21T10:29:10.668053+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:55.095131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":424,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.699930+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:29:10.699930+0000","last_peered":"2024-09-21T10:29:10.699930+0000","last_clean":"2024-09-21T10:29:10.699930+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:29:10.699930+0000","last_undegraded":"2024-09-21T10:29:10.699930+0000","last_fullsized":"2024-09-21T10:29:10.699930+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":464,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.700383+0000","last_change":"2024-09-21T10:25:48.766755+0000","last_active":"2024-09-21T10:29:10.700383+0000","last_peered":"2024-09-21T10:29:10.700383+0000","last_clean":"2024-09-21T10:29:10.700383+0000","last_became_active":"2024-09-21T10:25:48.766298+0000","last_became_peered":"2024-09-21T10:25:48.766298+0000","last_unstale":"2024-09-21T10:29:10.700383+0000","last_undegraded":"2024-09-21T10:29:10.700383+0000","last_fullsized":"2024-09-21T10:29:10.700383+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:18.788447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":10,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-21T10:29:18.324289+0000","last_change":"2024-09-21T10:29:18.323747+0000","last_active":"2024-09-21T10:29:18.324289+0000","last_peered":"2024-09-21T10:29:18.324289+0000","last_clean":"2024-09-21T10:29:18.324289+0000","last_became_active":"2024-09-21T10:29:18.323433+0000","last_became_peered":"2024-09-21T10:29:18.323433+0000","last_unstale":"2024-09-21T10:29:18.324289+0000","last_undegraded":"2024-09-21T10:29:18.324289+0000","last_fullsized":"2024-09-21T10:29:18.324289+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:50.727742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":670,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.274504+0000","last_change":"2024-09-21T10:25:48.757672+0000","last_active":"2024-09-21T10:29:04.274504+0000","last_peered":"2024-09-21T10:29:04.274504+0000","last_clean":"2024-09-21T10:29:04.274504+0000","last_became_active":"2024-09-21T10:25:48.757296+0000","last_became_peered":"2024-09-21T10:25:48.757296+0000","last_unstale":"2024-09-21T10:29:04.274504+0000","last_undegraded":"2024-09-21T10:29:04.274504+0000","last_fullsized":"2024-09-21T10:29:04.274504+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:31.244809+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:31.244809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:32.654430+0000","scrub_duration":0.0015828649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1163,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.271728+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:29:04.271728+0000","last_peered":"2024-09-21T10:29:04.271728+0000","last_clean":"2024-09-21T10:29:04.271728+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:29:04.271728+0000","last_undegraded":"2024-09-21T10:29:04.271728+0000","last_fullsized":"2024-09-21T10:29:04.271728+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":133,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312086+0000","last_change":"2024-09-21T10:25:48.753400+0000","last_active":"2024-09-21T10:29:17.312086+0000","last_peered":"2024-09-21T10:29:17.312086+0000","last_clean":"2024-09-21T10:29:17.312086+0000","last_became_active":"2024-09-21T10:25:48.753088+0000","last_became_peered":"2024-09-21T10:25:48.753088+0000","last_unstale":"2024-09-21T10:29:17.312086+0000","last_undegraded":"2024-09-21T10:29:17.312086+0000","last_fullsized":"2024-09-21T10:29:17.312086+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:49.107566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":423,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.700154+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:29:10.700154+0000","last_peered":"2024-09-21T10:29:10.700154+0000","last_clean":"2024-09-21T10:29:10.700154+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:29:10.700154+0000","last_undegraded":"2024-09-21T10:29:10.700154+0000","last_fullsized":"2024-09-21T10:29:10.700154+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":10,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-21T10:29:18.324067+0000","last_change":"2024-09-21T10:29:18.323540+0000","last_active":"2024-09-21T10:29:18.324067+0000","last_peered":"2024-09-21T10:29:18.324067+0000","last_clean":"2024-09-21T10:29:18.324067+0000","last_became_active":"2024-09-21T10:29:18.323125+0000","last_became_peered":"2024-09-21T10:29:18.323125+0000","last_unstale":"2024-09-21T10:29:18.324067+0000","last_undegraded":"2024-09-21T10:29:18.324067+0000","last_fullsized":"2024-09-21T10:29:18.324067+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:33.821481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":505,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.317477+0000","last_change":"2024-09-21T10:24:24.828950+0000","last_active":"2024-09-21T10:29:17.317477+0000","last_peered":"2024-09-21T10:29:17.317477+0000","last_clean":"2024-09-21T10:29:17.317477+0000","last_became_active":"2024-09-21T10:24:24.828550+0000","last_became_peered":"2024-09-21T10:24:24.828550+0000","last_unstale":"2024-09-21T10:29:17.317477+0000","last_undegraded":"2024-09-21T10:29:17.317477+0000","last_fullsized":"2024-09-21T10:29:17.317477+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:29.313198+0000","scrub_duration":0.001370488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":368,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.275524+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:29:04.275524+0000","last_peered":"2024-09-21T10:29:04.275524+0000","last_clean":"2024-09-21T10:29:04.275524+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:29:04.275524+0000","last_undegraded":"2024-09-21T10:29:04.275524+0000","last_fullsized":"2024-09-21T10:29:04.275524+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":1043,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.700265+0000","last_change":"2024-09-21T10:25:48.760586+0000","last_active":"2024-09-21T10:29:10.700265+0000","last_peered":"2024-09-21T10:29:10.700265+0000","last_clean":"2024-09-21T10:29:10.700265+0000","last_became_active":"2024-09-21T10:25:48.760042+0000","last_became_peered":"2024-09-21T10:25:48.760042+0000","last_unstale":"2024-09-21T10:29:10.700265+0000","last_undegraded":"2024-09-21T10:29:10.700265+0000","last_fullsized":"2024-09-21T10:29:10.700265+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:41.526062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":459,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.699547+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:29:10.699547+0000","last_peered":"2024-09-21T10:29:10.699547+0000","last_clean":"2024-09-21T10:29:10.699547+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:29:10.699547+0000","last_undegraded":"2024-09-21T10:29:10.699547+0000","last_fullsized":"2024-09-21T10:29:10.699547+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":10,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-21T10:29:18.322447+0000","last_change":"2024-09-21T10:29:18.320870+0000","last_active":"2024-09-21T10:29:18.322447+0000","last_peered":"2024-09-21T10:29:18.322447+0000","last_clean":"2024-09-21T10:29:18.322447+0000","last_became_active":"2024-09-21T10:29:18.320565+0000","last_became_peered":"2024-09-21T10:29:18.320565+0000","last_unstale":"2024-09-21T10:29:18.322447+0000","last_undegraded":"2024-09-21T10:29:18.322447+0000","last_fullsized":"2024-09-21T10:29:18.322447+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:51.308850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":520,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.317250+0000","last_change":"2024-09-21T10:25:48.752455+0000","last_active":"2024-09-21T10:29:17.317250+0000","last_peered":"2024-09-21T10:29:17.317250+0000","last_clean":"2024-09-21T10:29:17.317250+0000","last_became_active":"2024-09-21T10:25:48.752056+0000","last_became_peered":"2024-09-21T10:25:48.752056+0000","last_unstale":"2024-09-21T10:29:17.317250+0000","last_undegraded":"2024-09-21T10:29:17.317250+0000","last_fullsized":"2024-09-21T10:29:17.317250+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:15.886186+0000","scrub_duration":0.0018106349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":441,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.700210+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:29:10.700210+0000","last_peered":"2024-09-21T10:29:10.700210+0000","last_clean":"2024-09-21T10:29:10.700210+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:29:10.700210+0000","last_undegraded":"2024-09-21T10:29:10.700210+0000","last_fullsized":"2024-09-21T10:29:10.700210+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":1004,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.313003+0000","last_change":"2024-09-21T10:25:48.755967+0000","last_active":"2024-09-21T10:29:17.313003+0000","last_peered":"2024-09-21T10:29:17.313003+0000","last_clean":"2024-09-21T10:29:17.313003+0000","last_became_active":"2024-09-21T10:25:48.755037+0000","last_became_peered":"2024-09-21T10:25:48.755037+0000","last_unstale":"2024-09-21T10:29:17.313003+0000","last_undegraded":"2024-09-21T10:29:17.313003+0000","last_fullsized":"2024-09-21T10:29:17.313003+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:26.414898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":290,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603766+0000","last_change":"2024-09-21T10:25:48.767902+0000","last_active":"2024-09-21T10:29:10.603766+0000","last_peered":"2024-09-21T10:29:10.603766+0000","last_clean":"2024-09-21T10:29:10.603766+0000","last_became_active":"2024-09-21T10:25:48.765443+0000","last_became_peered":"2024-09-21T10:25:48.765443+0000","last_unstale":"2024-09-21T10:29:10.603766+0000","last_undegraded":"2024-09-21T10:29:10.603766+0000","last_fullsized":"2024-09-21T10:29:10.603766+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:26.622825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":10,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-21T10:29:18.323183+0000","last_change":"2024-09-21T10:29:18.321928+0000","last_active":"2024-09-21T10:29:18.323183+0000","last_peered":"2024-09-21T10:29:18.323183+0000","last_clean":"2024-09-21T10:29:18.323183+0000","last_became_active":"2024-09-21T10:29:18.321303+0000","last_became_peered":"2024-09-21T10:29:18.321303+0000","last_unstale":"2024-09-21T10:29:18.323183+0000","last_undegraded":"2024-09-21T10:29:18.323183+0000","last_fullsized":"2024-09-21T10:29:18.323183+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:25.083294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":657,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312303+0000","last_change":"2024-09-21T10:24:26.255840+0000","last_active":"2024-09-21T10:29:17.312303+0000","last_peered":"2024-09-21T10:29:17.312303+0000","last_clean":"2024-09-21T10:29:17.312303+0000","last_became_active":"2024-09-21T10:24:24.825549+0000","last_became_peered":"2024-09-21T10:24:24.825549+0000","last_unstale":"2024-09-21T10:29:17.312303+0000","last_undegraded":"2024-09-21T10:29:17.312303+0000","last_fullsized":"2024-09-21T10:29:17.312303+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:26.255753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.255753+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:25.638856+0000","scrub_duration":0.0016223839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":760,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.286279+0000","last_change":"2024-09-21T10:25:48.758590+0000","last_active":"2024-09-21T10:29:10.286279+0000","last_peered":"2024-09-21T10:29:10.286279+0000","last_clean":"2024-09-21T10:29:10.286279+0000","last_became_active":"2024-09-21T10:25:48.758189+0000","last_became_peered":"2024-09-21T10:25:48.758189+0000","last_unstale":"2024-09-21T10:29:10.286279+0000","last_undegraded":"2024-09-21T10:29:10.286279+0000","last_fullsized":"2024-09-21T10:29:10.286279+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:24.126237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1095,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.700368+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:29:10.700368+0000","last_peered":"2024-09-21T10:29:10.700368+0000","last_clean":"2024-09-21T10:29:10.700368+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:29:10.700368+0000","last_undegraded":"2024-09-21T10:29:10.700368+0000","last_fullsized":"2024-09-21T10:29:10.700368+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":756,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285435+0000","last_change":"2024-09-21T10:25:48.754966+0000","last_active":"2024-09-21T10:29:10.285435+0000","last_peered":"2024-09-21T10:29:10.285435+0000","last_clean":"2024-09-21T10:29:10.285435+0000","last_became_active":"2024-09-21T10:25:48.754152+0000","last_became_peered":"2024-09-21T10:25:48.754152+0000","last_unstale":"2024-09-21T10:29:10.285435+0000","last_undegraded":"2024-09-21T10:29:10.285435+0000","last_fullsized":"2024-09-21T10:29:10.285435+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:33:28.795713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"0'0","reported_seq":10,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-21T10:29:18.318939+0000","last_change":"2024-09-21T10:29:18.318036+0000","last_active":"2024-09-21T10:29:18.318939+0000","last_peered":"2024-09-21T10:29:18.318939+0000","last_clean":"2024-09-21T10:29:18.318939+0000","last_became_active":"2024-09-21T10:29:18.317716+0000","last_became_peered":"2024-09-21T10:29:18.317716+0000","last_unstale":"2024-09-21T10:29:18.318939+0000","last_undegraded":"2024-09-21T10:29:18.318939+0000","last_fullsized":"2024-09-21T10:29:18.318939+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:58.738776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":729,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285987+0000","last_change":"2024-09-21T10:24:26.926717+0000","last_active":"2024-09-21T10:29:10.285987+0000","last_peered":"2024-09-21T10:29:10.285987+0000","last_clean":"2024-09-21T10:29:10.285987+0000","last_became_active":"2024-09-21T10:24:24.828205+0000","last_became_peered":"2024-09-21T10:24:24.828205+0000","last_unstale":"2024-09-21T10:29:10.285987+0000","last_undegraded":"2024-09-21T10:29:10.285987+0000","last_fullsized":"2024-09-21T10:29:10.285987+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:26.926661+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.926661+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:47.946104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":111,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603937+0000","last_change":"2024-09-21T10:25:48.762565+0000","last_active":"2024-09-21T10:29:10.603937+0000","last_peered":"2024-09-21T10:29:10.603937+0000","last_clean":"2024-09-21T10:29:10.603937+0000","last_became_active":"2024-09-21T10:25:48.761324+0000","last_became_peered":"2024-09-21T10:25:48.761324+0000","last_unstale":"2024-09-21T10:29:10.603937+0000","last_undegraded":"2024-09-21T10:29:10.603937+0000","last_fullsized":"2024-09-21T10:29:10.603937+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:58.885790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":732,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.317600+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:29:17.317600+0000","last_peered":"2024-09-21T10:29:17.317600+0000","last_clean":"2024-09-21T10:29:17.317600+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:29:17.317600+0000","last_undegraded":"2024-09-21T10:29:17.317600+0000","last_fullsized":"2024-09-21T10:29:17.317600+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":394,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311539+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:29:17.311539+0000","last_peered":"2024-09-21T10:29:17.311539+0000","last_clean":"2024-09-21T10:29:17.311539+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:29:17.311539+0000","last_undegraded":"2024-09-21T10:29:17.311539+0000","last_fullsized":"2024-09-21T10:29:17.311539+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":418,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312165+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:29:17.312165+0000","last_peered":"2024-09-21T10:29:17.312165+0000","last_clean":"2024-09-21T10:29:17.312165+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:29:17.312165+0000","last_undegraded":"2024-09-21T10:29:17.312165+0000","last_fullsized":"2024-09-21T10:29:17.312165+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":128,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311978+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:29:17.311978+0000","last_peered":"2024-09-21T10:29:17.311978+0000","last_clean":"2024-09-21T10:29:17.311978+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:29:17.311978+0000","last_undegraded":"2024-09-21T10:29:17.311978+0000","last_fullsized":"2024-09-21T10:29:17.311978+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":119,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.271646+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:29:04.271646+0000","last_peered":"2024-09-21T10:29:04.271646+0000","last_clean":"2024-09-21T10:29:04.271646+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:29:04.271646+0000","last_undegraded":"2024-09-21T10:29:04.271646+0000","last_fullsized":"2024-09-21T10:29:04.271646+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":101,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.312811+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:29:17.312811+0000","last_peered":"2024-09-21T10:29:17.312811+0000","last_clean":"2024-09-21T10:29:17.312811+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:29:17.312811+0000","last_undegraded":"2024-09-21T10:29:17.312811+0000","last_fullsized":"2024-09-21T10:29:17.312811+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":127,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.273084+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:29:04.273084+0000","last_peered":"2024-09-21T10:29:04.273084+0000","last_clean":"2024-09-21T10:29:04.273084+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:29:04.273084+0000","last_undegraded":"2024-09-21T10:29:04.273084+0000","last_fullsized":"2024-09-21T10:29:04.273084+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":668,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285438+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:29:10.285438+0000","last_peered":"2024-09-21T10:29:10.285438+0000","last_clean":"2024-09-21T10:29:10.285438+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:29:10.285438+0000","last_undegraded":"2024-09-21T10:29:10.285438+0000","last_fullsized":"2024-09-21T10:29:10.285438+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":288,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603975+0000","last_change":"2024-09-21T10:25:48.765786+0000","last_active":"2024-09-21T10:29:10.603975+0000","last_peered":"2024-09-21T10:29:10.603975+0000","last_clean":"2024-09-21T10:29:10.603975+0000","last_became_active":"2024-09-21T10:25:48.764011+0000","last_became_peered":"2024-09-21T10:25:48.764011+0000","last_unstale":"2024-09-21T10:29:10.603975+0000","last_undegraded":"2024-09-21T10:29:10.603975+0000","last_fullsized":"2024-09-21T10:29:10.603975+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:13.352641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":728,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285352+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:29:10.285352+0000","last_peered":"2024-09-21T10:29:10.285352+0000","last_clean":"2024-09-21T10:29:10.285352+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:29:10.285352+0000","last_undegraded":"2024-09-21T10:29:10.285352+0000","last_fullsized":"2024-09-21T10:29:10.285352+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":436,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.317503+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:29:17.317503+0000","last_peered":"2024-09-21T10:29:17.317503+0000","last_clean":"2024-09-21T10:29:17.317503+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:29:17.317503+0000","last_undegraded":"2024-09-21T10:29:17.317503+0000","last_fullsized":"2024-09-21T10:29:17.317503+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":756,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285473+0000","last_change":"2024-09-21T10:25:48.755341+0000","last_active":"2024-09-21T10:29:10.285473+0000","last_peered":"2024-09-21T10:29:10.285473+0000","last_clean":"2024-09-21T10:29:10.285473+0000","last_became_active":"2024-09-21T10:25:48.754620+0000","last_became_peered":"2024-09-21T10:25:48.754620+0000","last_unstale":"2024-09-21T10:29:10.285473+0000","last_undegraded":"2024-09-21T10:29:10.285473+0000","last_fullsized":"2024-09-21T10:29:10.285473+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:57:21.296549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":321,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603816+0000","last_change":"2024-09-21T10:25:48.765683+0000","last_active":"2024-09-21T10:29:10.603816+0000","last_peered":"2024-09-21T10:29:10.603816+0000","last_clean":"2024-09-21T10:29:10.603816+0000","last_became_active":"2024-09-21T10:25:48.765281+0000","last_became_peered":"2024-09-21T10:25:48.765281+0000","last_unstale":"2024-09-21T10:29:10.603816+0000","last_undegraded":"2024-09-21T10:29:10.603816+0000","last_fullsized":"2024-09-21T10:29:10.603816+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:41.798683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":443,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.699917+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:29:10.699917+0000","last_peered":"2024-09-21T10:29:10.699917+0000","last_clean":"2024-09-21T10:29:10.699917+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:29:10.699917+0000","last_undegraded":"2024-09-21T10:29:10.699917+0000","last_fullsized":"2024-09-21T10:29:10.699917+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":735,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285635+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:29:10.285635+0000","last_peered":"2024-09-21T10:29:10.285635+0000","last_clean":"2024-09-21T10:29:10.285635+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:29:10.285635+0000","last_undegraded":"2024-09-21T10:29:10.285635+0000","last_fullsized":"2024-09-21T10:29:10.285635+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":425,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311781+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:29:17.311781+0000","last_peered":"2024-09-21T10:29:17.311781+0000","last_clean":"2024-09-21T10:29:17.311781+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:29:17.311781+0000","last_undegraded":"2024-09-21T10:29:17.311781+0000","last_fullsized":"2024-09-21T10:29:17.311781+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":734,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.668112+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:29:10.668112+0000","last_peered":"2024-09-21T10:29:10.668112+0000","last_clean":"2024-09-21T10:29:10.668112+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:29:10.668112+0000","last_undegraded":"2024-09-21T10:29:10.668112+0000","last_fullsized":"2024-09-21T10:29:10.668112+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":81,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603932+0000","last_change":"2024-09-21T10:25:48.767385+0000","last_active":"2024-09-21T10:29:10.603932+0000","last_peered":"2024-09-21T10:29:10.603932+0000","last_clean":"2024-09-21T10:29:10.603932+0000","last_became_active":"2024-09-21T10:25:48.764077+0000","last_became_peered":"2024-09-21T10:25:48.764077+0000","last_unstale":"2024-09-21T10:29:10.603932+0000","last_undegraded":"2024-09-21T10:29:10.603932+0000","last_fullsized":"2024-09-21T10:29:10.603932+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:06:51.969987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":117,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311928+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:29:17.311928+0000","last_peered":"2024-09-21T10:29:17.311928+0000","last_clean":"2024-09-21T10:29:17.311928+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:29:17.311928+0000","last_undegraded":"2024-09-21T10:29:17.311928+0000","last_fullsized":"2024-09-21T10:29:17.311928+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":359,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.271995+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:29:04.271995+0000","last_peered":"2024-09-21T10:29:04.271995+0000","last_clean":"2024-09-21T10:29:04.271995+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:29:04.271995+0000","last_undegraded":"2024-09-21T10:29:04.271995+0000","last_fullsized":"2024-09-21T10:29:04.271995+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":132,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311632+0000","last_change":"2024-09-21T10:25:48.753233+0000","last_active":"2024-09-21T10:29:17.311632+0000","last_peered":"2024-09-21T10:29:17.311632+0000","last_clean":"2024-09-21T10:29:17.311632+0000","last_became_active":"2024-09-21T10:25:48.752932+0000","last_became_peered":"2024-09-21T10:25:48.752932+0000","last_unstale":"2024-09-21T10:29:17.311632+0000","last_undegraded":"2024-09-21T10:29:17.311632+0000","last_fullsized":"2024-09-21T10:29:17.311632+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:49.456475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":337,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-21T10:29:04.276071+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:29:04.276071+0000","last_peered":"2024-09-21T10:29:04.276071+0000","last_clean":"2024-09-21T10:29:04.276071+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:29:04.276071+0000","last_undegraded":"2024-09-21T10:29:04.276071+0000","last_fullsized":"2024-09-21T10:29:04.276071+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":759,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.285697+0000","last_change":"2024-09-21T10:25:48.755262+0000","last_active":"2024-09-21T10:29:10.285697+0000","last_peered":"2024-09-21T10:29:10.285697+0000","last_clean":"2024-09-21T10:29:10.285697+0000","last_became_active":"2024-09-21T10:25:48.754746+0000","last_became_peered":"2024-09-21T10:25:48.754746+0000","last_unstale":"2024-09-21T10:29:10.285697+0000","last_undegraded":"2024-09-21T10:29:10.285697+0000","last_fullsized":"2024-09-21T10:29:10.285697+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:28:17.312418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":710,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.286051+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:29:10.286051+0000","last_peered":"2024-09-21T10:29:10.286051+0000","last_clean":"2024-09-21T10:29:10.286051+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:29:10.286051+0000","last_undegraded":"2024-09-21T10:29:10.286051+0000","last_fullsized":"2024-09-21T10:29:10.286051+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":117,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-21T10:29:17.311142+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:29:17.311142+0000","last_peered":"2024-09-21T10:29:17.311142+0000","last_clean":"2024-09-21T10:29:17.311142+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:29:17.311142+0000","last_undegraded":"2024-09-21T10:29:17.311142+0000","last_fullsized":"2024-09-21T10:29:17.311142+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":113,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-21T10:29:10.603847+0000","last_change":"2024-09-21T10:25:48.764108+0000","last_active":"2024-09-21T10:29:10.603847+0000","last_peered":"2024-09-21T10:29:10.603847+0000","last_clean":"2024-09-21T10:29:10.603847+0000","last_became_active":"2024-09-21T10:25:48.761361+0000","last_became_peered":"2024-09-21T10:25:48.761361+0000","last_unstale":"2024-09-21T10:29:10.603847+0000","last_undegraded":"2024-09-21T10:29:10.603847+0000","last_fullsized":"2024-09-21T10:29:10.603847+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:49.188529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":3},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28818,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234476,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":861168,"kb_used_data":9696,"kb_used_omap":1,"kb_used_meta":851454,"kb_avail":92883984,"statfs":{"total":95995035648,"available":95113199616,"internally_reserved":0,"allocated":9928704,"data_stored":7792181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":871889355},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496114,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":704172,"kb_used_data":5276,"kb_used_omap":1,"kb_used_meta":698878,"kb_avail":93040980,"statfs":{"total":95995035648,"available":95273963520,"internally_reserved":0,"allocated":5402624,"data_stored":3270401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":715651527},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569404013,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":561164,"kb_used_data":5824,"kb_used_omap":7,"kb_used_meta":555320,"kb_avail":93183988,"statfs":{"total":95995035648,"available":95420403712,"internally_reserved":0,"allocated":5963776,"data_stored":3839615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":568648600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019387,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":658220,"kb_used_data":5280,"kb_used_omap":1,"kb_used_meta":652926,"kb_avail":93086932,"statfs":{"total":95995035648,"available":95321018368,"internally_reserved":0,"allocated":5406720,"data_stored":3270458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":668596678},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435570,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":989892,"kb_used_data":5816,"kb_used_omap":5,"kb_used_meta":984058,"kb_avail":92755260,"statfs":{"total":95995035648,"available":94981386240,"internally_reserved":0,"allocated":5955584,"data_stored":3839046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":1007675432},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599097,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":795204,"kb_used_data":5496,"kb_used_omap":7,"kb_used_meta":789688,"kb_avail":92949948,"statfs":{"total":95995035648,"available":95180746752,"internally_reserved":0,"allocated":5627904,"data_stored":3511584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":808640973},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762624,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":789308,"kb_used_data":9328,"kb_used_omap":7,"kb_used_meta":779960,"kb_avail":92955844,"statfs":{"total":95995035648,"available":95186784256,"internally_reserved":0,"allocated":9551872,"data_stored":7442638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":798679293},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":389,"seq":1670742278187,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":154180,"kb_used_data":9336,"kb_used_omap":12,"kb_used_meta":144819,"kb_avail":93590972,"statfs":{"total":95995035648,"available":95837155328,"internally_reserved":0,"allocated":9560064,"data_stored":7443496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12595,"internal_metadata":148295373},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11001,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:29:23.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:23.236 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:23.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:23.337 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:23.359 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:23.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:23.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:23.461 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:23.488 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:23.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:23.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:23.589 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:23.612 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:23.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:23.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:23.713 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:23.735 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:23.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:23.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:23.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:23 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:29:23.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:23 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:29:23.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:23 smithi032 ceph-mon[105748]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T10:29:23.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:23 smithi032 ceph-mon[105748]: pgmap v1327: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:23.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:23 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:29:23.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:23 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:29:23.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:23 smithi032 ceph-mon[101299]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T10:29:23.825 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:23 smithi032 ceph-mon[101299]: pgmap v1327: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:23.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:23.859 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:23.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:23.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:23.960 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:23.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:23.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:23.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:23 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:29:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:23 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:29:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:23 smithi045 ceph-mon[95976]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T10:29:24.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:23 smithi045 ceph-mon[95976]: pgmap v1327: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:24.089 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:24.112 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:24.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:24.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:24.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:24.240 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:24.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:24.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:24.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:24.364 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:24.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:24.466 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:24.488 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:24.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:24.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:24.589 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:24.616 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:24.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:24.718 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:24.744 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:24.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:24.846 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:24.855 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:29:24.855 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:29:24.855 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:29:24.855 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:29:24.855 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:29:24.869 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:24.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:24.970 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:24.992 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:24.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:24.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:25.094 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:25.116 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:25.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:25.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:25.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:25.240 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:25.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:25.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:25.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:25.364 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:25.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:25.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:25.465 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:25.492 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:25.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:25.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:25.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:25.616 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:25.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:25.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:25.718 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:25.740 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:25.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:25.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:25.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:25.865 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:25.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:25.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:25.966 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:25.993 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:25.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:25.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:26.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:25 smithi045 ceph-mon[95976]: pgmap v1328: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:26.095 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:26.118 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:26.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:26.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:29:26.148 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:25 smithi032 ceph-mon[105748]: pgmap v1328: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:26.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:25 smithi032 ceph-mon[101299]: pgmap v1328: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:26.171 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:29:26.220 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:26.236 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:29:26.247 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:26.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:26.349 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:26.374 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:26.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:26.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:26.476 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:26.498 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:26.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:26.600 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:26.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:26.623 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:26.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:26.724 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:26.750 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:26.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:26.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:26.852 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:26.874 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:26.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:26.976 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:26.998 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:26.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:26.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:27.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:27.127 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:27.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:27.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:27.229 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:27.255 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:27.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:27.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:27.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:27.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:27.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:27.483 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:27.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:27.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:27.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:27.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:27.634 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:27.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:27.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:27.736 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:27.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:27.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:27.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:29:27.911 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:29:27.913 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:29:27.913 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:29:27.929 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:29:27.930 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:29:27.942 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:27.946 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:29:27.946 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:29:27.947 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:27 smithi032 ceph-mon[105748]: pgmap v1329: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:27.947 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:27 smithi032 ceph-mon[101299]: pgmap v1329: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:27.963 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:29:27.963 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:29:27.968 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:27.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:27.980 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:29:27.980 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:29:27.996 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:29:27.997 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:29:28.014 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:29:28.014 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:29:28.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:27 smithi045 ceph-mon[95976]: pgmap v1329: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:28.044 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:28.070 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:28.096 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:28.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:28.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:28.198 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:28.221 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:28.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:28.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:28.266 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:28.274 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:28.322 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:28.349 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:28.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:28.451 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:28.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:28.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:28.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:28.577 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:28.600 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:28.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:28.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:28.701 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:28.724 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:28.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:28.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:28.753 INFO:teuthology.orchestra.run.smithi032.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-09-21T10:29:28.825 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:28.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:28.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:28.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:28.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:29.009 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:29.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:29.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:28 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/362667716' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:28 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/696127222' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:29:29.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:28 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:29:29.041 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:28 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/362667716' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:29.042 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:28 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/696127222' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:29:29.042 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:28 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:29:29.042 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:28 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/362667716' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:29.042 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:28 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/696127222' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:29:29.042 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:28 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:29:29.111 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:29.138 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:29.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:29.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:29.239 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:29.264 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:29.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:29.365 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:29.389 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:29.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:29.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:29.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:29.516 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:29.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:29.618 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:29.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:29.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:29.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:29.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:29.750 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:29:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:29:29] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.51.0" 2024-09-21T10:29:29.765 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:29.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:29.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:29.867 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:29.893 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:29.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:29.995 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:30.022 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:30.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:30.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:30.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:29 smithi045 ceph-mon[95976]: pgmap v1330: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:29 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:29:30.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:29 smithi045 ceph-mon[95976]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T10:29:30.076 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:29 smithi032 ceph-mon[105748]: pgmap v1330: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:30.076 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:29 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:29:30.076 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:29 smithi032 ceph-mon[105748]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T10:29:30.076 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:29 smithi032 ceph-mon[101299]: pgmap v1330: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:30.076 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:29 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:29:30.076 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:29 smithi032 ceph-mon[101299]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T10:29:30.124 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:30.151 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:30.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:30.253 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:30.280 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:30.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:30.381 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:30.405 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:30.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:30.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:30.507 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:30.529 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:30.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:30.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:30.631 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:30.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:30.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:30.759 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:30.782 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:30.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:30.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:30.824 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:29:30.883 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:30.905 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:30.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:30.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:30.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:30 smithi032 ceph-mon[105748]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:29:30.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:30 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:29:30.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:30 smithi032 ceph-mon[101299]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:29:30.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:30 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:29:31.007 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:31.035 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:31.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:31.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:31.102 DEBUG:teuthology.orchestra.run.smithi032:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-09-21T10:29:31.104 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:29:31.104 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:29:31.136 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:31.158 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:31.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:31.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:31.176 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:hints = 1 2024-09-21T10:29:31.176 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:29:31.176 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object prefix: benchmark_data_smithi032_297285 2024-09-21T10:29:31.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:29:31.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:29:31.260 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:30 smithi045 ceph-mon[95976]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:29:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:30 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:29:31.297 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:31.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:31.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:31.399 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:31.437 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:31.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:31.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:31.538 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:31.574 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:31.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:31.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:31.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:31.715 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:31.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:31.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:31.817 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:31.852 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:31.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:31.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:31.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:31.992 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:31.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:31.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:32.093 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:32.131 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:32.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:32.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:32.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 1 16 3794 3778 236.112 236.125 0.00354983 0.00422012 2024-09-21T10:29:32.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:31 smithi032 ceph-mon[101299]: pgmap v1332: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:32.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:31 smithi032 ceph-mon[105748]: pgmap v1332: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:32.232 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:32.269 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:32.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:32.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:31 smithi045 ceph-mon[95976]: pgmap v1332: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-21T10:29:32.357 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:32.371 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:32.406 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:32.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:32.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:32.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:32.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:32.546 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:32.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:32.647 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:32.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:32.686 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:32.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:32.786 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:32.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:32.814 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:32.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:32.915 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:32.951 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:32.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:32.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:33.053 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:33.095 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:33.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:33.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:33.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 2 15 7129 7114 222.296 208.5 0.00679221 0.0044894 2024-09-21T10:29:33.196 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:33.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:33.235 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:33.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:33.336 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:33.375 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:33.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:33.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:33.476 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:33.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:33.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:33.611 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:33.645 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:33.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:33.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:33.747 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:33.788 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:33.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:33.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:33.890 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:33.923 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:33.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:34.024 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:34.047 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:29:34.047 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:29:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:34.058 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:34.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:34.063 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:33 smithi032 ceph-mon[101299]: pgmap v1333: 99 pgs: 99 active+clean; 38 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s wr, 53 op/s 2024-09-21T10:29:34.064 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:34.064 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:34.064 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:33 smithi032 ceph-mon[105748]: pgmap v1333: 99 pgs: 99 active+clean; 38 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s wr, 53 op/s 2024-09-21T10:29:34.064 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:34.064 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:34.159 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:34.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 3 16 9978 9962 207.526 178 0.00399654 0.00480917 2024-09-21T10:29:34.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:34.195 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:34.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:34.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:33 smithi045 ceph-mon[95976]: pgmap v1333: 99 pgs: 99 active+clean; 38 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s wr, 53 op/s 2024-09-21T10:29:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:34.296 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:34.336 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:34.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:34.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:34.438 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:34.477 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:34.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:34.578 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:34.617 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:34.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:34.719 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:34.756 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:34.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:34.822 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:29:34.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:34.890 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:34.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:34.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:34.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:35.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:35.030 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:35.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:35.131 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:35.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:35.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:35.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:35.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 4 16 13051 13035 203.655 192.062 0.00278426 0.00490373 2024-09-21T10:29:35.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3936759368' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:29:35.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3936759368' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:29:35.273 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:35.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3936759368' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:29:35.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:35.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:35.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:35.422 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:35.459 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:35.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:35.561 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:35.598 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:35.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:35.699 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:35.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:35.737 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:35.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:35.838 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:35.876 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:35.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:35.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:35.978 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:36.013 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:36.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:36.081 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:35 smithi032 ceph-mon[101299]: pgmap v1334: 99 pgs: 99 active+clean; 464 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2024-09-21T10:29:36.081 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:35 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:29:36.081 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:35 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3936759368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:29:36.081 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:35 smithi032 ceph-mon[101299]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T10:29:36.081 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:35 smithi032 ceph-mon[105748]: pgmap v1334: 99 pgs: 99 active+clean; 464 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2024-09-21T10:29:36.082 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:35 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:29:36.082 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:35 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3936759368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:29:36.082 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:35 smithi032 ceph-mon[105748]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T10:29:36.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:36.150 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:36.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:36.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:36.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 5 16 16125 16109 201.346 192.125 0.00825023 0.00495984 2024-09-21T10:29:36.251 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:36.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:35 smithi045 ceph-mon[95976]: pgmap v1334: 99 pgs: 99 active+clean; 464 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2024-09-21T10:29:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:35 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:29:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:35 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3936759368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:29:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:35 smithi045 ceph-mon[95976]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T10:29:36.288 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:36.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:36.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:36.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:36.427 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:36.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:36.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:36.528 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:36.566 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:36.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:36.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:36.668 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:36.705 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:36.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:36.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:36.807 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:36.843 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:36.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:36.944 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:36.982 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:36.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:36.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:37.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:37.123 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:37.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:37.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 6 16 19042 19026 198.171 182.312 0.00535627 0.00503977 2024-09-21T10:29:37.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:37.267 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:37.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:37.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:37.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:37.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:37.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:37.513 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:37.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:37.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:37.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:37.603 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:29:37.603 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:29:37.640 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:29:37.640 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:29:37.651 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:37.680 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:29:37.680 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:29:37.689 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:37.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:37.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:37.713 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:29:37.713 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:29:37.750 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:29:37.750 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:29:37.780 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:29:37.780 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:29:37.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:37.815 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:29:37.815 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:29:37.829 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:37.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:37.850 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:29:37.850 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:29:37.932 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:37.968 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:37.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:37.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:38.070 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:38.092 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:37 smithi032 ceph-mon[101299]: pgmap v1336: 99 pgs: 99 active+clean; 647 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:29:38.092 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:37 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/842692841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:38.092 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:37 smithi032 ceph-mon[105748]: pgmap v1336: 99 pgs: 99 active+clean; 647 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:29:38.092 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:37 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/842692841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:38.107 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:38.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:38.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 7 16 21415 21399 191.044 148.312 0.00216823 0.00518211 2024-09-21T10:29:38.209 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:38.246 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:38.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:38.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:38.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:37 smithi045 ceph-mon[95976]: pgmap v1336: 99 pgs: 99 active+clean; 647 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:29:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:37 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/842692841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:38.348 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:38.385 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:38.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:38.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:38.487 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:38.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:38.526 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:38.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:38.627 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:38.664 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:38.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:38.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:38.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:38.793 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:38.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:38.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:38.894 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:38.931 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:38.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:38.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:38.932 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:29:39.033 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:39.055 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:39.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:39.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:39.157 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:39.177 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 8 16 23701 23685 185.021 142.875 0.00284728 0.0053989 2024-09-21T10:29:39.193 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:39.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:39.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:39.295 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:39.335 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:39.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:39.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:39.437 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:39.474 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:39.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:39.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:39.576 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:39.611 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:39.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:39.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:39.712 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:39.747 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:39.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:39.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:39.820 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:29:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:29:39] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.51.0" 2024-09-21T10:29:39.849 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:39.887 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:39.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:39.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:39.989 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:40.024 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:40.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:40.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:40.162 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:40.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:40.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 9 16 25857 25841 179.434 134.75 0.00364958 0.00556803 2024-09-21T10:29:40.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:39 smithi032 ceph-mon[101299]: pgmap v1337: 99 pgs: 99 active+clean; 767 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:29:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:39 smithi032 ceph-mon[105748]: pgmap v1337: 99 pgs: 99 active+clean; 767 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:29:40.265 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:40.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:39 smithi045 ceph-mon[95976]: pgmap v1337: 99 pgs: 99 active+clean; 767 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:29:40.300 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:40.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:40.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:40.402 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:40.442 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:40.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:40.544 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:40.566 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:40.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:40.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:40.667 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:40.704 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:40.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:40.806 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:40.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:40.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:40.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:40.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:40.981 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:40.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:41.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:41.119 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:41.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:41.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:41.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 10 16 27639 27623 172.626 111.375 0.00166127 0.00575291 2024-09-21T10:29:41.221 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:41.244 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:41.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:41.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:41.345 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:41.367 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:41.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:41.468 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:41.506 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:41.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:41.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:41.608 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:41.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:41.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:41.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:41.749 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:41.788 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:41.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:41.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:41.890 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:41.931 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:41.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:41.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:42.033 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:42.071 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:42.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:42.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:42.173 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:42.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 11 16 29597 29581 168.057 122.375 0.00888757 0.00594509 2024-09-21T10:29:42.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:41 smithi032 ceph-mon[101299]: pgmap v1338: 99 pgs: 99 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2024-09-21T10:29:42.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:41 smithi032 ceph-mon[105748]: pgmap v1338: 99 pgs: 99 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2024-09-21T10:29:42.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:42.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:41 smithi045 ceph-mon[95976]: pgmap v1338: 99 pgs: 99 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2024-09-21T10:29:42.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:42.337 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:42.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:42.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:42.439 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:42.477 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:42.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:42.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:42.579 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:42.617 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:42.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:42.674 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:29:42.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:42.758 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:42.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:42.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:42.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:42.894 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:42.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:42.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:42.996 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:43.058 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:43.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:43.156 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:29:43.160 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:43.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 12 16 31493 31477 163.926 118.5 0.00438947 0.00609508 2024-09-21T10:29:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:43.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:43.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:43.287 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:43.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:43.345 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:43.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:43.447 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:43.483 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:43.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:43.585 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:43.626 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:43.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:43.729 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:43.764 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:29:43.765 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:29:43.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:43.767 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:43.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:43.868 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:43.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:43.895 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:43.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:43.996 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:44.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:44.034 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:44.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:44.135 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:44.170 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:44.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:44.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 13 16 33520 33504 161.061 126.688 0.00540204 0.00620383 2024-09-21T10:29:44.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:43 smithi032 ceph-mon[101299]: pgmap v1339: 99 pgs: 99 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s 2024-09-21T10:29:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:43 smithi032 ceph-mon[105748]: pgmap v1339: 99 pgs: 99 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s 2024-09-21T10:29:44.272 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:44.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:43 smithi045 ceph-mon[95976]: pgmap v1339: 99 pgs: 99 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s 2024-09-21T10:29:44.304 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:44.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:44.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:44.405 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:44.443 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:44.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:44.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:44.544 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:44.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:44.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:44.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:44.682 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:44.706 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:44.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:44.808 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:44.847 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:44.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:44.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:44.948 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:44.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:44.990 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:44.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:45.091 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:45.128 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:45.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:45.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:45.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 14 16 35140 35124 156.788 101.25 0.00195783 0.00634922 2024-09-21T10:29:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:44 smithi032 ceph-mon[101299]: from='client.46166 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:29:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:44 smithi032 ceph-mon[105748]: from='client.46166 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:29:45.229 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:45.252 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:45.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:45.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:44 smithi045 ceph-mon[95976]: from='client.46166 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:29:45.353 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:45.392 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:45.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:45.493 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:45.528 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:45.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:45.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:45.631 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:45.670 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:45.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:45.773 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:45.812 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:45.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:45.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:45.914 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:45.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:45.951 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:45.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:46.052 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:46.090 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:46.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:46.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 15 16 36944 36928 153.852 112.75 0.0030019 0.00649482 2024-09-21T10:29:46.191 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:45 smithi032 ceph-mon[105748]: pgmap v1340: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 153 MiB/s wr, 2.45k op/s 2024-09-21T10:29:46.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:45 smithi032 ceph-mon[101299]: pgmap v1340: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 153 MiB/s wr, 2.45k op/s 2024-09-21T10:29:46.229 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:46.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:46.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:45 smithi045 ceph-mon[95976]: pgmap v1340: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 153 MiB/s wr, 2.45k op/s 2024-09-21T10:29:46.331 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:46.367 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:46.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:46.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:46.469 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:46.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:46.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:46.603 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:46.629 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:46.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:46.661 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:29:46.730 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:46.766 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:46.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:46.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:46.868 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:46.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:46.908 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:46.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:47.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:47.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:47.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:47.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:47.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:47.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 16 16 38767 38751 151.357 113.938 0.00649856 0.00660248 2024-09-21T10:29:47.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:47.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:47.289 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:47.327 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:47.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:47.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:47.429 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:47.468 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:47.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:47.538 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1339,"stamp":"2024-09-21T10:29:42.694661+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1658853349,"num_objects":25267,"num_object_clones":0,"num_object_copies":75801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25267,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":26147,"num_write_kb":1627921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":285,"num_bytes_recovered":16782981,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27913,"ondisk_log_size":27913,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10469740,"kb_used_data":4874668,"kb_used_omap":45,"kb_used_meta":5594962,"kb_avail":739491476,"statfs":{"total":767960285184,"available":757239271424,"internally_reserved":0,"allocated":4991660032,"data_stored":4974511383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46417,"internal_metadata":5729241775},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,7,6,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1619001344,"num_objects":24704,"num_object_clones":0,"num_object_copies":74112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":24704,"num_write_kb":1581056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004550"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":728,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827470+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:29:34.827470+0000","last_peered":"2024-09-21T10:29:34.827470+0000","last_clean":"2024-09-21T10:29:34.827470+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:29:34.827470+0000","last_undegraded":"2024-09-21T10:29:34.827470+0000","last_fullsized":"2024-09-21T10:29:34.827470+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":429,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.258054+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:29:29.258054+0000","last_peered":"2024-09-21T10:29:29.258054+0000","last_clean":"2024-09-21T10:29:29.258054+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:29:29.258054+0000","last_undegraded":"2024-09-21T10:29:29.258054+0000","last_fullsized":"2024-09-21T10:29:29.258054+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":125,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.751451+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:29:28.751451+0000","last_peered":"2024-09-21T10:29:28.751451+0000","last_clean":"2024-09-21T10:29:28.751451+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:29:28.751451+0000","last_undegraded":"2024-09-21T10:29:28.751451+0000","last_fullsized":"2024-09-21T10:29:28.751451+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":94,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.832622+0000","last_change":"2024-09-21T10:25:48.765404+0000","last_active":"2024-09-21T10:29:34.832622+0000","last_peered":"2024-09-21T10:29:34.832622+0000","last_clean":"2024-09-21T10:29:34.832622+0000","last_became_active":"2024-09-21T10:25:48.764230+0000","last_became_peered":"2024-09-21T10:25:48.764230+0000","last_unstale":"2024-09-21T10:29:34.832622+0000","last_undegraded":"2024-09-21T10:29:34.832622+0000","last_fullsized":"2024-09-21T10:29:34.832622+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:38.551941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":95,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.833115+0000","last_change":"2024-09-21T10:25:48.764391+0000","last_active":"2024-09-21T10:29:34.833115+0000","last_peered":"2024-09-21T10:29:34.833115+0000","last_clean":"2024-09-21T10:29:34.833115+0000","last_became_active":"2024-09-21T10:25:48.763375+0000","last_became_peered":"2024-09-21T10:25:48.763375+0000","last_unstale":"2024-09-21T10:29:34.833115+0000","last_undegraded":"2024-09-21T10:29:34.833115+0000","last_fullsized":"2024-09-21T10:29:34.833115+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:03.457716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":400,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.257324+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:29:29.257324+0000","last_peered":"2024-09-21T10:29:29.257324+0000","last_clean":"2024-09-21T10:29:29.257324+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:29:29.257324+0000","last_undegraded":"2024-09-21T10:29:29.257324+0000","last_fullsized":"2024-09-21T10:29:29.257324+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":748,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827678+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:29:34.827678+0000","last_peered":"2024-09-21T10:29:34.827678+0000","last_clean":"2024-09-21T10:29:34.827678+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:29:34.827678+0000","last_undegraded":"2024-09-21T10:29:34.827678+0000","last_fullsized":"2024-09-21T10:29:34.827678+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":496,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.257382+0000","last_change":"2024-09-21T10:25:48.752712+0000","last_active":"2024-09-21T10:29:29.257382+0000","last_peered":"2024-09-21T10:29:29.257382+0000","last_clean":"2024-09-21T10:29:29.257382+0000","last_became_active":"2024-09-21T10:25:48.752204+0000","last_became_peered":"2024-09-21T10:25:48.752204+0000","last_unstale":"2024-09-21T10:29:29.257382+0000","last_undegraded":"2024-09-21T10:29:29.257382+0000","last_fullsized":"2024-09-21T10:29:29.257382+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:40.068520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"424'1707","reported_seq":1720,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:41.025735+0000","last_change":"2024-09-21T10:29:18.322432+0000","last_active":"2024-09-21T10:29:41.025735+0000","last_peered":"2024-09-21T10:29:41.025735+0000","last_clean":"2024-09-21T10:29:41.025735+0000","last_became_active":"2024-09-21T10:29:18.322116+0000","last_became_peered":"2024-09-21T10:29:18.322116+0000","last_unstale":"2024-09-21T10:29:41.025735+0000","last_undegraded":"2024-09-21T10:29:41.025735+0000","last_fullsized":"2024-09-21T10:29:41.025735+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1707,"log_dups_size":0,"ondisk_log_size":1707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:35.158580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111869952,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1707,"num_write_kb":109248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":131,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.258530+0000","last_change":"2024-09-21T10:25:48.755730+0000","last_active":"2024-09-21T10:29:29.258530+0000","last_peered":"2024-09-21T10:29:29.258530+0000","last_clean":"2024-09-21T10:29:29.258530+0000","last_became_active":"2024-09-21T10:25:48.755344+0000","last_became_peered":"2024-09-21T10:25:48.755344+0000","last_unstale":"2024-09-21T10:29:29.258530+0000","last_undegraded":"2024-09-21T10:29:29.258530+0000","last_fullsized":"2024-09-21T10:29:29.258530+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:37.725090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":134,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750563+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:29:28.750563+0000","last_peered":"2024-09-21T10:29:28.750563+0000","last_clean":"2024-09-21T10:29:28.750563+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:29:28.750563+0000","last_undegraded":"2024-09-21T10:29:28.750563+0000","last_fullsized":"2024-09-21T10:29:28.750563+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"424'1356","reported_seq":1370,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:39.069414+0000","last_change":"2024-09-21T10:29:18.320824+0000","last_active":"2024-09-21T10:29:39.069414+0000","last_peered":"2024-09-21T10:29:39.069414+0000","last_clean":"2024-09-21T10:29:39.069414+0000","last_became_active":"2024-09-21T10:29:18.320520+0000","last_became_peered":"2024-09-21T10:29:18.320520+0000","last_unstale":"2024-09-21T10:29:39.069414+0000","last_undegraded":"2024-09-21T10:29:39.069414+0000","last_fullsized":"2024-09-21T10:29:39.069414+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1356,"log_dups_size":0,"ondisk_log_size":1356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:23.749267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88866816,"num_objects":1356,"num_object_clones":0,"num_object_copies":4068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1356,"num_write_kb":86784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":103,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.258586+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:29:29.258586+0000","last_peered":"2024-09-21T10:29:29.258586+0000","last_clean":"2024-09-21T10:29:29.258586+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:29:29.258586+0000","last_undegraded":"2024-09-21T10:29:29.258586+0000","last_fullsized":"2024-09-21T10:29:29.258586+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":122,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750160+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:29:28.750160+0000","last_peered":"2024-09-21T10:29:28.750160+0000","last_clean":"2024-09-21T10:29:28.750160+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:29:28.750160+0000","last_undegraded":"2024-09-21T10:29:28.750160+0000","last_fullsized":"2024-09-21T10:29:28.750160+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"424'1921","reported_seq":1942,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:42.209567+0000","last_change":"2024-09-21T10:29:18.320476+0000","last_active":"2024-09-21T10:29:42.209567+0000","last_peered":"2024-09-21T10:29:42.209567+0000","last_clean":"2024-09-21T10:29:42.209567+0000","last_became_active":"2024-09-21T10:29:18.320167+0000","last_became_peered":"2024-09-21T10:29:18.320167+0000","last_unstale":"2024-09-21T10:29:42.209567+0000","last_undegraded":"2024-09-21T10:29:42.209567+0000","last_fullsized":"2024-09-21T10:29:42.209567+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1921,"log_dups_size":0,"ondisk_log_size":1921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:38:15.611579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125894656,"num_objects":1921,"num_object_clones":0,"num_object_copies":5763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1921,"num_write_kb":122944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":425,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.258225+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:29:29.258225+0000","last_peered":"2024-09-21T10:29:29.258225+0000","last_clean":"2024-09-21T10:29:29.258225+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:29:29.258225+0000","last_undegraded":"2024-09-21T10:29:29.258225+0000","last_fullsized":"2024-09-21T10:29:29.258225+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":445,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.081842+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:29:34.081842+0000","last_peered":"2024-09-21T10:29:34.081842+0000","last_clean":"2024-09-21T10:29:34.081842+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:29:34.081842+0000","last_undegraded":"2024-09-21T10:29:34.081842+0000","last_fullsized":"2024-09-21T10:29:34.081842+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"424'1496","reported_seq":1517,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:39.624558+0000","last_change":"2024-09-21T10:29:18.321766+0000","last_active":"2024-09-21T10:29:39.624558+0000","last_peered":"2024-09-21T10:29:39.624558+0000","last_clean":"2024-09-21T10:29:39.624558+0000","last_became_active":"2024-09-21T10:29:18.321306+0000","last_became_peered":"2024-09-21T10:29:18.321306+0000","last_unstale":"2024-09-21T10:29:39.624558+0000","last_undegraded":"2024-09-21T10:29:39.624558+0000","last_fullsized":"2024-09-21T10:29:39.624558+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1496,"log_dups_size":0,"ondisk_log_size":1496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:39:43.157433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98041856,"num_objects":1496,"num_object_clones":0,"num_object_copies":4488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1496,"num_write_kb":95744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":359,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.751037+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:29:28.751037+0000","last_peered":"2024-09-21T10:29:28.751037+0000","last_clean":"2024-09-21T10:29:28.751037+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:29:28.751037+0000","last_undegraded":"2024-09-21T10:29:28.751037+0000","last_fullsized":"2024-09-21T10:29:28.751037+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":285,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.836427+0000","last_change":"2024-09-21T10:25:48.763096+0000","last_active":"2024-09-21T10:29:34.836427+0000","last_peered":"2024-09-21T10:29:34.836427+0000","last_clean":"2024-09-21T10:29:34.836427+0000","last_became_active":"2024-09-21T10:25:48.761327+0000","last_became_peered":"2024-09-21T10:25:48.761327+0000","last_unstale":"2024-09-21T10:29:34.836427+0000","last_undegraded":"2024-09-21T10:29:34.836427+0000","last_fullsized":"2024-09-21T10:29:34.836427+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:23.154333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"424'1718","reported_seq":1739,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:41.012467+0000","last_change":"2024-09-21T10:29:18.323639+0000","last_active":"2024-09-21T10:29:41.012467+0000","last_peered":"2024-09-21T10:29:41.012467+0000","last_clean":"2024-09-21T10:29:41.012467+0000","last_became_active":"2024-09-21T10:29:18.323318+0000","last_became_peered":"2024-09-21T10:29:18.323318+0000","last_unstale":"2024-09-21T10:29:41.012467+0000","last_undegraded":"2024-09-21T10:29:41.012467+0000","last_fullsized":"2024-09-21T10:29:41.012467+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1718,"log_dups_size":0,"ondisk_log_size":1718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:27:31.319819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112590848,"num_objects":1718,"num_object_clones":0,"num_object_copies":5154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1718,"num_write_kb":109952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":710,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.256910+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:29:29.256910+0000","last_peered":"2024-09-21T10:29:29.256910+0000","last_clean":"2024-09-21T10:29:29.256910+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:29:29.256910+0000","last_undegraded":"2024-09-21T10:29:29.256910+0000","last_fullsized":"2024-09-21T10:29:29.256910+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":789,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827017+0000","last_change":"2024-09-21T10:25:48.755999+0000","last_active":"2024-09-21T10:29:34.827017+0000","last_peered":"2024-09-21T10:29:34.827017+0000","last_clean":"2024-09-21T10:29:34.827017+0000","last_became_active":"2024-09-21T10:25:48.754257+0000","last_became_peered":"2024-09-21T10:25:48.754257+0000","last_unstale":"2024-09-21T10:29:34.827017+0000","last_undegraded":"2024-09-21T10:29:34.827017+0000","last_fullsized":"2024-09-21T10:29:34.827017+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:06.025116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":122,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750280+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:29:28.750280+0000","last_peered":"2024-09-21T10:29:28.750280+0000","last_clean":"2024-09-21T10:29:28.750280+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:29:28.750280+0000","last_undegraded":"2024-09-21T10:29:28.750280+0000","last_fullsized":"2024-09-21T10:29:28.750280+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"424'1602","reported_seq":1623,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:39.631957+0000","last_change":"2024-09-21T10:29:18.322964+0000","last_active":"2024-09-21T10:29:39.631957+0000","last_peered":"2024-09-21T10:29:39.631957+0000","last_clean":"2024-09-21T10:29:39.631957+0000","last_became_active":"2024-09-21T10:29:18.322677+0000","last_became_peered":"2024-09-21T10:29:18.322677+0000","last_unstale":"2024-09-21T10:29:39.631957+0000","last_undegraded":"2024-09-21T10:29:39.631957+0000","last_fullsized":"2024-09-21T10:29:39.631957+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1602,"log_dups_size":0,"ondisk_log_size":1602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:06.684046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104988672,"num_objects":1602,"num_object_clones":0,"num_object_copies":4806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1602,"num_write_kb":102528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":483,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750473+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:29:28.750473+0000","last_peered":"2024-09-21T10:29:28.750473+0000","last_clean":"2024-09-21T10:29:28.750473+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:29:28.750473+0000","last_undegraded":"2024-09-21T10:29:28.750473+0000","last_fullsized":"2024-09-21T10:29:28.750473+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":435,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.081853+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:29:34.081853+0000","last_peered":"2024-09-21T10:29:34.081853+0000","last_clean":"2024-09-21T10:29:34.081853+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:29:34.081853+0000","last_undegraded":"2024-09-21T10:29:34.081853+0000","last_fullsized":"2024-09-21T10:29:34.081853+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":122,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750652+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:29:28.750652+0000","last_peered":"2024-09-21T10:29:28.750652+0000","last_clean":"2024-09-21T10:29:28.750652+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:29:28.750652+0000","last_undegraded":"2024-09-21T10:29:28.750652+0000","last_fullsized":"2024-09-21T10:29:28.750652+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"424'1492","reported_seq":1506,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:39.070637+0000","last_change":"2024-09-21T10:29:18.322112+0000","last_active":"2024-09-21T10:29:39.070637+0000","last_peered":"2024-09-21T10:29:39.070637+0000","last_clean":"2024-09-21T10:29:39.070637+0000","last_became_active":"2024-09-21T10:29:18.321784+0000","last_became_peered":"2024-09-21T10:29:18.321784+0000","last_unstale":"2024-09-21T10:29:39.070637+0000","last_undegraded":"2024-09-21T10:29:39.070637+0000","last_fullsized":"2024-09-21T10:29:39.070637+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1492,"log_dups_size":0,"ondisk_log_size":1492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:10.300972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97779712,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1492,"num_write_kb":95488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":122,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.749963+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:29:28.749963+0000","last_peered":"2024-09-21T10:29:28.749963+0000","last_clean":"2024-09-21T10:29:28.749963+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:29:28.749963+0000","last_undegraded":"2024-09-21T10:29:28.749963+0000","last_fullsized":"2024-09-21T10:29:28.749963+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":122,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750857+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:29:28.750857+0000","last_peered":"2024-09-21T10:29:28.750857+0000","last_clean":"2024-09-21T10:29:28.750857+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:29:28.750857+0000","last_undegraded":"2024-09-21T10:29:28.750857+0000","last_fullsized":"2024-09-21T10:29:28.750857+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"424'1455","reported_seq":1469,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:38.983311+0000","last_change":"2024-09-21T10:29:18.319916+0000","last_active":"2024-09-21T10:29:38.983311+0000","last_peered":"2024-09-21T10:29:38.983311+0000","last_clean":"2024-09-21T10:29:38.983311+0000","last_became_active":"2024-09-21T10:29:18.319395+0000","last_became_peered":"2024-09-21T10:29:18.319395+0000","last_unstale":"2024-09-21T10:29:38.983311+0000","last_undegraded":"2024-09-21T10:29:38.983311+0000","last_fullsized":"2024-09-21T10:29:38.983311+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1455,"log_dups_size":0,"ondisk_log_size":1455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:46.166177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95354880,"num_objects":1455,"num_object_clones":0,"num_object_copies":4365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1455,"num_write_kb":93120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":661,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.751139+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:29:28.751139+0000","last_peered":"2024-09-21T10:29:28.751139+0000","last_clean":"2024-09-21T10:29:28.751139+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:29:28.751139+0000","last_undegraded":"2024-09-21T10:29:28.751139+0000","last_fullsized":"2024-09-21T10:29:28.751139+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":756,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827180+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:29:34.827180+0000","last_peered":"2024-09-21T10:29:34.827180+0000","last_clean":"2024-09-21T10:29:34.827180+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:29:34.827180+0000","last_undegraded":"2024-09-21T10:29:34.827180+0000","last_fullsized":"2024-09-21T10:29:34.827180+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":449,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.829365+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:29:34.829365+0000","last_peered":"2024-09-21T10:29:34.829365+0000","last_clean":"2024-09-21T10:29:34.829365+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:29:34.829365+0000","last_undegraded":"2024-09-21T10:29:34.829365+0000","last_fullsized":"2024-09-21T10:29:34.829365+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"424'1445","reported_seq":1459,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:39.071196+0000","last_change":"2024-09-21T10:29:18.321006+0000","last_active":"2024-09-21T10:29:39.071196+0000","last_peered":"2024-09-21T10:29:39.071196+0000","last_clean":"2024-09-21T10:29:39.071196+0000","last_became_active":"2024-09-21T10:29:18.320643+0000","last_became_peered":"2024-09-21T10:29:18.320643+0000","last_unstale":"2024-09-21T10:29:39.071196+0000","last_undegraded":"2024-09-21T10:29:39.071196+0000","last_fullsized":"2024-09-21T10:29:39.071196+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1445,"log_dups_size":0,"ondisk_log_size":1445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:44.782234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94699520,"num_objects":1445,"num_object_clones":0,"num_object_copies":4335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1445,"num_write_kb":92480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":709,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.257449+0000","last_change":"2024-09-21T10:25:48.754725+0000","last_active":"2024-09-21T10:29:29.257449+0000","last_peered":"2024-09-21T10:29:29.257449+0000","last_clean":"2024-09-21T10:29:29.257449+0000","last_became_active":"2024-09-21T10:25:48.754341+0000","last_became_peered":"2024-09-21T10:25:48.754341+0000","last_unstale":"2024-09-21T10:29:29.257449+0000","last_undegraded":"2024-09-21T10:29:29.257449+0000","last_fullsized":"2024-09-21T10:29:29.257449+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:25.191555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:25.191555+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:08.981054+0000","scrub_duration":0.0015044590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":738,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750523+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:29:28.750523+0000","last_peered":"2024-09-21T10:29:28.750523+0000","last_clean":"2024-09-21T10:29:28.750523+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:29:28.750523+0000","last_undegraded":"2024-09-21T10:29:28.750523+0000","last_fullsized":"2024-09-21T10:29:28.750523+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":688,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827027+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:29:34.827027+0000","last_peered":"2024-09-21T10:29:34.827027+0000","last_clean":"2024-09-21T10:29:34.827027+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:29:34.827027+0000","last_undegraded":"2024-09-21T10:29:34.827027+0000","last_fullsized":"2024-09-21T10:29:34.827027+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":401,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.258126+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:29:29.258126+0000","last_peered":"2024-09-21T10:29:29.258126+0000","last_clean":"2024-09-21T10:29:29.258126+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:29:29.258126+0000","last_undegraded":"2024-09-21T10:29:29.258126+0000","last_fullsized":"2024-09-21T10:29:29.258126+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"424'1898","reported_seq":1919,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:42.194877+0000","last_change":"2024-09-21T10:29:18.319178+0000","last_active":"2024-09-21T10:29:42.194877+0000","last_peered":"2024-09-21T10:29:42.194877+0000","last_clean":"2024-09-21T10:29:42.194877+0000","last_became_active":"2024-09-21T10:29:18.318926+0000","last_became_peered":"2024-09-21T10:29:18.318926+0000","last_unstale":"2024-09-21T10:29:42.194877+0000","last_undegraded":"2024-09-21T10:29:42.194877+0000","last_fullsized":"2024-09-21T10:29:42.194877+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1898,"log_dups_size":0,"ondisk_log_size":1898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:22.626306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124387328,"num_objects":1898,"num_object_clones":0,"num_object_copies":5694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1898,"num_write_kb":121472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":705,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.830528+0000","last_change":"2024-09-21T10:24:27.359111+0000","last_active":"2024-09-21T10:29:34.830528+0000","last_peered":"2024-09-21T10:29:34.830528+0000","last_clean":"2024-09-21T10:29:34.830528+0000","last_became_active":"2024-09-21T10:24:24.825850+0000","last_became_peered":"2024-09-21T10:24:24.825850+0000","last_unstale":"2024-09-21T10:29:34.830528+0000","last_undegraded":"2024-09-21T10:29:34.830528+0000","last_fullsized":"2024-09-21T10:29:34.830528+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:27.359065+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:24:27.359065+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:06.871319+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1082,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.286315+0000","last_change":"2024-09-21T10:25:48.755667+0000","last_active":"2024-09-21T10:29:29.286315+0000","last_peered":"2024-09-21T10:29:29.286315+0000","last_clean":"2024-09-21T10:29:29.286315+0000","last_became_active":"2024-09-21T10:25:48.755060+0000","last_became_peered":"2024-09-21T10:25:48.755060+0000","last_unstale":"2024-09-21T10:29:29.286315+0000","last_undegraded":"2024-09-21T10:29:29.286315+0000","last_fullsized":"2024-09-21T10:29:29.286315+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:09.230030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":287,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.833062+0000","last_change":"2024-09-21T10:25:48.754659+0000","last_active":"2024-09-21T10:29:34.833062+0000","last_peered":"2024-09-21T10:29:34.833062+0000","last_clean":"2024-09-21T10:29:34.833062+0000","last_became_active":"2024-09-21T10:25:48.754230+0000","last_became_peered":"2024-09-21T10:25:48.754230+0000","last_unstale":"2024-09-21T10:29:34.833062+0000","last_undegraded":"2024-09-21T10:29:34.833062+0000","last_fullsized":"2024-09-21T10:29:34.833062+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:12.555868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":94,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.832539+0000","last_change":"2024-09-21T10:25:48.761249+0000","last_active":"2024-09-21T10:29:34.832539+0000","last_peered":"2024-09-21T10:29:34.832539+0000","last_clean":"2024-09-21T10:29:34.832539+0000","last_became_active":"2024-09-21T10:25:48.759359+0000","last_became_peered":"2024-09-21T10:25:48.759359+0000","last_unstale":"2024-09-21T10:29:34.832539+0000","last_undegraded":"2024-09-21T10:29:34.832539+0000","last_fullsized":"2024-09-21T10:29:34.832539+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:33.510936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"424'1808","reported_seq":1822,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:41.007927+0000","last_change":"2024-09-21T10:29:18.320919+0000","last_active":"2024-09-21T10:29:41.007927+0000","last_peered":"2024-09-21T10:29:41.007927+0000","last_clean":"2024-09-21T10:29:41.007927+0000","last_became_active":"2024-09-21T10:29:18.320597+0000","last_became_peered":"2024-09-21T10:29:18.320597+0000","last_unstale":"2024-09-21T10:29:41.007927+0000","last_undegraded":"2024-09-21T10:29:41.007927+0000","last_fullsized":"2024-09-21T10:29:41.007927+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1808,"log_dups_size":0,"ondisk_log_size":1808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:30.147686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118489088,"num_objects":1808,"num_object_clones":0,"num_object_copies":5424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1808,"num_write_kb":115712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":725,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.257826+0000","last_change":"2024-09-21T10:25:48.755509+0000","last_active":"2024-09-21T10:29:29.257826+0000","last_peered":"2024-09-21T10:29:29.257826+0000","last_clean":"2024-09-21T10:29:29.257826+0000","last_became_active":"2024-09-21T10:25:48.754558+0000","last_became_peered":"2024-09-21T10:25:48.754558+0000","last_unstale":"2024-09-21T10:29:29.257826+0000","last_undegraded":"2024-09-21T10:29:29.257826+0000","last_fullsized":"2024-09-21T10:29:29.257826+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:11.262503+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1288,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.832749+0000","last_change":"2024-09-21T10:25:48.753359+0000","last_active":"2024-09-21T10:29:34.832749+0000","last_peered":"2024-09-21T10:29:34.832749+0000","last_clean":"2024-09-21T10:29:34.832749+0000","last_became_active":"2024-09-21T10:25:48.752653+0000","last_became_peered":"2024-09-21T10:25:48.752653+0000","last_unstale":"2024-09-21T10:29:34.832749+0000","last_undegraded":"2024-09-21T10:29:34.832749+0000","last_fullsized":"2024-09-21T10:29:34.832749+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:55.095131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":436,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.828549+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:29:34.828549+0000","last_peered":"2024-09-21T10:29:34.828549+0000","last_clean":"2024-09-21T10:29:34.828549+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:29:34.828549+0000","last_undegraded":"2024-09-21T10:29:34.828549+0000","last_fullsized":"2024-09-21T10:29:34.828549+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":476,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.829454+0000","last_change":"2024-09-21T10:25:48.766755+0000","last_active":"2024-09-21T10:29:34.829454+0000","last_peered":"2024-09-21T10:29:34.829454+0000","last_clean":"2024-09-21T10:29:34.829454+0000","last_became_active":"2024-09-21T10:25:48.766298+0000","last_became_peered":"2024-09-21T10:25:48.766298+0000","last_unstale":"2024-09-21T10:29:34.829454+0000","last_undegraded":"2024-09-21T10:29:34.829454+0000","last_fullsized":"2024-09-21T10:29:34.829454+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:18.788447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"424'1501","reported_seq":1515,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:39.070463+0000","last_change":"2024-09-21T10:29:18.323747+0000","last_active":"2024-09-21T10:29:39.070463+0000","last_peered":"2024-09-21T10:29:39.070463+0000","last_clean":"2024-09-21T10:29:39.070463+0000","last_became_active":"2024-09-21T10:29:18.323433+0000","last_became_peered":"2024-09-21T10:29:18.323433+0000","last_unstale":"2024-09-21T10:29:39.070463+0000","last_undegraded":"2024-09-21T10:29:39.070463+0000","last_fullsized":"2024-09-21T10:29:39.070463+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1501,"log_dups_size":0,"ondisk_log_size":1501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:50.727742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98369536,"num_objects":1501,"num_object_clones":0,"num_object_copies":4503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1501,"num_write_kb":96064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":676,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750925+0000","last_change":"2024-09-21T10:25:48.757672+0000","last_active":"2024-09-21T10:29:28.750925+0000","last_peered":"2024-09-21T10:29:28.750925+0000","last_clean":"2024-09-21T10:29:28.750925+0000","last_became_active":"2024-09-21T10:25:48.757296+0000","last_became_peered":"2024-09-21T10:25:48.757296+0000","last_unstale":"2024-09-21T10:29:28.750925+0000","last_undegraded":"2024-09-21T10:29:28.750925+0000","last_fullsized":"2024-09-21T10:29:28.750925+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:31.244809+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:31.244809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:32.654430+0000","scrub_duration":0.0015828649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1169,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.751417+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:29:28.751417+0000","last_peered":"2024-09-21T10:29:28.751417+0000","last_clean":"2024-09-21T10:29:28.751417+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:29:28.751417+0000","last_undegraded":"2024-09-21T10:29:28.751417+0000","last_fullsized":"2024-09-21T10:29:28.751417+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":137,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750557+0000","last_change":"2024-09-21T10:25:48.753400+0000","last_active":"2024-09-21T10:29:28.750557+0000","last_peered":"2024-09-21T10:29:28.750557+0000","last_clean":"2024-09-21T10:29:28.750557+0000","last_became_active":"2024-09-21T10:25:48.753088+0000","last_became_peered":"2024-09-21T10:25:48.753088+0000","last_unstale":"2024-09-21T10:29:28.750557+0000","last_undegraded":"2024-09-21T10:29:28.750557+0000","last_fullsized":"2024-09-21T10:29:28.750557+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:49.107566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":435,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.828458+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:29:34.828458+0000","last_peered":"2024-09-21T10:29:34.828458+0000","last_clean":"2024-09-21T10:29:34.828458+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:29:34.828458+0000","last_undegraded":"2024-09-21T10:29:34.828458+0000","last_fullsized":"2024-09-21T10:29:34.828458+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"424'1422","reported_seq":1436,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:39.280951+0000","last_change":"2024-09-21T10:29:18.323540+0000","last_active":"2024-09-21T10:29:39.280951+0000","last_peered":"2024-09-21T10:29:39.280951+0000","last_clean":"2024-09-21T10:29:39.280951+0000","last_became_active":"2024-09-21T10:29:18.323125+0000","last_became_peered":"2024-09-21T10:29:18.323125+0000","last_unstale":"2024-09-21T10:29:39.280951+0000","last_undegraded":"2024-09-21T10:29:39.280951+0000","last_fullsized":"2024-09-21T10:29:39.280951+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1422,"log_dups_size":0,"ondisk_log_size":1422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:33.821481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93192192,"num_objects":1422,"num_object_clones":0,"num_object_copies":4266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1422,"num_write_kb":91008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":509,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.257676+0000","last_change":"2024-09-21T10:24:24.828950+0000","last_active":"2024-09-21T10:29:29.257676+0000","last_peered":"2024-09-21T10:29:29.257676+0000","last_clean":"2024-09-21T10:29:29.257676+0000","last_became_active":"2024-09-21T10:24:24.828550+0000","last_became_peered":"2024-09-21T10:24:24.828550+0000","last_unstale":"2024-09-21T10:29:29.257676+0000","last_undegraded":"2024-09-21T10:29:29.257676+0000","last_fullsized":"2024-09-21T10:29:29.257676+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:29.313198+0000","scrub_duration":0.001370488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":374,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750847+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:29:28.750847+0000","last_peered":"2024-09-21T10:29:28.750847+0000","last_clean":"2024-09-21T10:29:28.750847+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:29:28.750847+0000","last_undegraded":"2024-09-21T10:29:28.750847+0000","last_fullsized":"2024-09-21T10:29:28.750847+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":1055,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.828751+0000","last_change":"2024-09-21T10:25:48.760586+0000","last_active":"2024-09-21T10:29:34.828751+0000","last_peered":"2024-09-21T10:29:34.828751+0000","last_clean":"2024-09-21T10:29:34.828751+0000","last_became_active":"2024-09-21T10:25:48.760042+0000","last_became_peered":"2024-09-21T10:25:48.760042+0000","last_unstale":"2024-09-21T10:29:34.828751+0000","last_undegraded":"2024-09-21T10:29:34.828751+0000","last_fullsized":"2024-09-21T10:29:34.828751+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:41.526062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":471,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.830015+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:29:34.830015+0000","last_peered":"2024-09-21T10:29:34.830015+0000","last_clean":"2024-09-21T10:29:34.830015+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:29:34.830015+0000","last_undegraded":"2024-09-21T10:29:34.830015+0000","last_fullsized":"2024-09-21T10:29:34.830015+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"424'1453","reported_seq":1465,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:39.073925+0000","last_change":"2024-09-21T10:29:18.320870+0000","last_active":"2024-09-21T10:29:39.073925+0000","last_peered":"2024-09-21T10:29:39.073925+0000","last_clean":"2024-09-21T10:29:39.073925+0000","last_became_active":"2024-09-21T10:29:18.320565+0000","last_became_peered":"2024-09-21T10:29:18.320565+0000","last_unstale":"2024-09-21T10:29:39.073925+0000","last_undegraded":"2024-09-21T10:29:39.073925+0000","last_fullsized":"2024-09-21T10:29:39.073925+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1453,"log_dups_size":0,"ondisk_log_size":1453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:51.308850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95223808,"num_objects":1453,"num_object_clones":0,"num_object_copies":4359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1453,"num_write_kb":92992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":524,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.257912+0000","last_change":"2024-09-21T10:25:48.752455+0000","last_active":"2024-09-21T10:29:29.257912+0000","last_peered":"2024-09-21T10:29:29.257912+0000","last_clean":"2024-09-21T10:29:29.257912+0000","last_became_active":"2024-09-21T10:25:48.752056+0000","last_became_peered":"2024-09-21T10:25:48.752056+0000","last_unstale":"2024-09-21T10:29:29.257912+0000","last_undegraded":"2024-09-21T10:29:29.257912+0000","last_fullsized":"2024-09-21T10:29:29.257912+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:15.886186+0000","scrub_duration":0.0018106349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":453,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.828664+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:29:34.828664+0000","last_peered":"2024-09-21T10:29:34.828664+0000","last_clean":"2024-09-21T10:29:34.828664+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:29:34.828664+0000","last_undegraded":"2024-09-21T10:29:34.828664+0000","last_fullsized":"2024-09-21T10:29:34.828664+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":1008,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.258342+0000","last_change":"2024-09-21T10:25:48.755967+0000","last_active":"2024-09-21T10:29:29.258342+0000","last_peered":"2024-09-21T10:29:29.258342+0000","last_clean":"2024-09-21T10:29:29.258342+0000","last_became_active":"2024-09-21T10:25:48.755037+0000","last_became_peered":"2024-09-21T10:25:48.755037+0000","last_unstale":"2024-09-21T10:29:29.258342+0000","last_undegraded":"2024-09-21T10:29:29.258342+0000","last_fullsized":"2024-09-21T10:29:29.258342+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:26.414898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":302,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.836109+0000","last_change":"2024-09-21T10:25:48.767902+0000","last_active":"2024-09-21T10:29:34.836109+0000","last_peered":"2024-09-21T10:29:34.836109+0000","last_clean":"2024-09-21T10:29:34.836109+0000","last_became_active":"2024-09-21T10:25:48.765443+0000","last_became_peered":"2024-09-21T10:25:48.765443+0000","last_unstale":"2024-09-21T10:29:34.836109+0000","last_undegraded":"2024-09-21T10:29:34.836109+0000","last_fullsized":"2024-09-21T10:29:34.836109+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:26.622825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"424'1494","reported_seq":1508,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:38.998206+0000","last_change":"2024-09-21T10:29:18.321928+0000","last_active":"2024-09-21T10:29:38.998206+0000","last_peered":"2024-09-21T10:29:38.998206+0000","last_clean":"2024-09-21T10:29:38.998206+0000","last_became_active":"2024-09-21T10:29:18.321303+0000","last_became_peered":"2024-09-21T10:29:18.321303+0000","last_unstale":"2024-09-21T10:29:38.998206+0000","last_undegraded":"2024-09-21T10:29:38.998206+0000","last_fullsized":"2024-09-21T10:29:38.998206+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1494,"log_dups_size":0,"ondisk_log_size":1494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:25.083294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97910784,"num_objects":1494,"num_object_clones":0,"num_object_copies":4482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1494,"num_write_kb":95616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":661,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750937+0000","last_change":"2024-09-21T10:24:26.255840+0000","last_active":"2024-09-21T10:29:28.750937+0000","last_peered":"2024-09-21T10:29:28.750937+0000","last_clean":"2024-09-21T10:29:28.750937+0000","last_became_active":"2024-09-21T10:24:24.825549+0000","last_became_peered":"2024-09-21T10:24:24.825549+0000","last_unstale":"2024-09-21T10:29:28.750937+0000","last_undegraded":"2024-09-21T10:29:28.750937+0000","last_fullsized":"2024-09-21T10:29:28.750937+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:26.255753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.255753+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:25.638856+0000","scrub_duration":0.0016223839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":772,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827796+0000","last_change":"2024-09-21T10:25:48.758590+0000","last_active":"2024-09-21T10:29:34.827796+0000","last_peered":"2024-09-21T10:29:34.827796+0000","last_clean":"2024-09-21T10:29:34.827796+0000","last_became_active":"2024-09-21T10:25:48.758189+0000","last_became_peered":"2024-09-21T10:25:48.758189+0000","last_unstale":"2024-09-21T10:29:34.827796+0000","last_undegraded":"2024-09-21T10:29:34.827796+0000","last_fullsized":"2024-09-21T10:29:34.827796+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:24.126237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1107,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.829570+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:29:34.829570+0000","last_peered":"2024-09-21T10:29:34.829570+0000","last_clean":"2024-09-21T10:29:34.829570+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:29:34.829570+0000","last_undegraded":"2024-09-21T10:29:34.829570+0000","last_fullsized":"2024-09-21T10:29:34.829570+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":768,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.826958+0000","last_change":"2024-09-21T10:25:48.754966+0000","last_active":"2024-09-21T10:29:34.826958+0000","last_peered":"2024-09-21T10:29:34.826958+0000","last_clean":"2024-09-21T10:29:34.826958+0000","last_became_active":"2024-09-21T10:25:48.754152+0000","last_became_peered":"2024-09-21T10:25:48.754152+0000","last_unstale":"2024-09-21T10:29:34.826958+0000","last_undegraded":"2024-09-21T10:29:34.826958+0000","last_fullsized":"2024-09-21T10:29:34.826958+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:33:28.795713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"424'1471","reported_seq":1485,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:39.078202+0000","last_change":"2024-09-21T10:29:18.318036+0000","last_active":"2024-09-21T10:29:39.078202+0000","last_peered":"2024-09-21T10:29:39.078202+0000","last_clean":"2024-09-21T10:29:39.078202+0000","last_became_active":"2024-09-21T10:29:18.317716+0000","last_became_peered":"2024-09-21T10:29:18.317716+0000","last_unstale":"2024-09-21T10:29:39.078202+0000","last_undegraded":"2024-09-21T10:29:39.078202+0000","last_fullsized":"2024-09-21T10:29:39.078202+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":1471,"log_dups_size":0,"ondisk_log_size":1471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:58.738776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96403456,"num_objects":1471,"num_object_clones":0,"num_object_copies":4413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1471,"num_write_kb":94144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":741,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827364+0000","last_change":"2024-09-21T10:24:26.926717+0000","last_active":"2024-09-21T10:29:34.827364+0000","last_peered":"2024-09-21T10:29:34.827364+0000","last_clean":"2024-09-21T10:29:34.827364+0000","last_became_active":"2024-09-21T10:24:24.828205+0000","last_became_peered":"2024-09-21T10:24:24.828205+0000","last_unstale":"2024-09-21T10:29:34.827364+0000","last_undegraded":"2024-09-21T10:29:34.827364+0000","last_fullsized":"2024-09-21T10:29:34.827364+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:26.926661+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.926661+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:47.946104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":123,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.833214+0000","last_change":"2024-09-21T10:25:48.762565+0000","last_active":"2024-09-21T10:29:34.833214+0000","last_peered":"2024-09-21T10:29:34.833214+0000","last_clean":"2024-09-21T10:29:34.833214+0000","last_became_active":"2024-09-21T10:25:48.761324+0000","last_became_peered":"2024-09-21T10:25:48.761324+0000","last_unstale":"2024-09-21T10:29:34.833214+0000","last_undegraded":"2024-09-21T10:29:34.833214+0000","last_fullsized":"2024-09-21T10:29:34.833214+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:58.885790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":736,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.258680+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:29:29.258680+0000","last_peered":"2024-09-21T10:29:29.258680+0000","last_clean":"2024-09-21T10:29:29.258680+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:29:29.258680+0000","last_undegraded":"2024-09-21T10:29:29.258680+0000","last_fullsized":"2024-09-21T10:29:29.258680+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":398,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750475+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:29:28.750475+0000","last_peered":"2024-09-21T10:29:28.750475+0000","last_clean":"2024-09-21T10:29:28.750475+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:29:28.750475+0000","last_undegraded":"2024-09-21T10:29:28.750475+0000","last_fullsized":"2024-09-21T10:29:28.750475+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":422,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750653+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:29:28.750653+0000","last_peered":"2024-09-21T10:29:28.750653+0000","last_clean":"2024-09-21T10:29:28.750653+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:29:28.750653+0000","last_undegraded":"2024-09-21T10:29:28.750653+0000","last_fullsized":"2024-09-21T10:29:28.750653+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":132,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750273+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:29:28.750273+0000","last_peered":"2024-09-21T10:29:28.750273+0000","last_clean":"2024-09-21T10:29:28.750273+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:29:28.750273+0000","last_undegraded":"2024-09-21T10:29:28.750273+0000","last_fullsized":"2024-09-21T10:29:28.750273+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":126,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:31.042813+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:29:31.042813+0000","last_peered":"2024-09-21T10:29:31.042813+0000","last_clean":"2024-09-21T10:29:31.042813+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:29:31.042813+0000","last_undegraded":"2024-09-21T10:29:31.042813+0000","last_fullsized":"2024-09-21T10:29:31.042813+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":105,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.258469+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:29:29.258469+0000","last_peered":"2024-09-21T10:29:29.258469+0000","last_clean":"2024-09-21T10:29:29.258469+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:29:29.258469+0000","last_undegraded":"2024-09-21T10:29:29.258469+0000","last_fullsized":"2024-09-21T10:29:29.258469+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":133,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.751315+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:29:28.751315+0000","last_peered":"2024-09-21T10:29:28.751315+0000","last_clean":"2024-09-21T10:29:28.751315+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:29:28.751315+0000","last_undegraded":"2024-09-21T10:29:28.751315+0000","last_fullsized":"2024-09-21T10:29:28.751315+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":681,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.826913+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:29:34.826913+0000","last_peered":"2024-09-21T10:29:34.826913+0000","last_clean":"2024-09-21T10:29:34.826913+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:29:34.826913+0000","last_undegraded":"2024-09-21T10:29:34.826913+0000","last_fullsized":"2024-09-21T10:29:34.826913+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":300,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.833348+0000","last_change":"2024-09-21T10:25:48.765786+0000","last_active":"2024-09-21T10:29:34.833348+0000","last_peered":"2024-09-21T10:29:34.833348+0000","last_clean":"2024-09-21T10:29:34.833348+0000","last_became_active":"2024-09-21T10:25:48.764011+0000","last_became_peered":"2024-09-21T10:25:48.764011+0000","last_unstale":"2024-09-21T10:29:34.833348+0000","last_undegraded":"2024-09-21T10:29:34.833348+0000","last_fullsized":"2024-09-21T10:29:34.833348+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:13.352641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":741,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.826853+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:29:34.826853+0000","last_peered":"2024-09-21T10:29:34.826853+0000","last_clean":"2024-09-21T10:29:34.826853+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:29:34.826853+0000","last_undegraded":"2024-09-21T10:29:34.826853+0000","last_fullsized":"2024-09-21T10:29:34.826853+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":440,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:29.258059+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:29:29.258059+0000","last_peered":"2024-09-21T10:29:29.258059+0000","last_clean":"2024-09-21T10:29:29.258059+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:29:29.258059+0000","last_undegraded":"2024-09-21T10:29:29.258059+0000","last_fullsized":"2024-09-21T10:29:29.258059+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":768,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827070+0000","last_change":"2024-09-21T10:25:48.755341+0000","last_active":"2024-09-21T10:29:34.827070+0000","last_peered":"2024-09-21T10:29:34.827070+0000","last_clean":"2024-09-21T10:29:34.827070+0000","last_became_active":"2024-09-21T10:25:48.754620+0000","last_became_peered":"2024-09-21T10:25:48.754620+0000","last_unstale":"2024-09-21T10:29:34.827070+0000","last_undegraded":"2024-09-21T10:29:34.827070+0000","last_fullsized":"2024-09-21T10:29:34.827070+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:57:21.296549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":333,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.832956+0000","last_change":"2024-09-21T10:25:48.765683+0000","last_active":"2024-09-21T10:29:34.832956+0000","last_peered":"2024-09-21T10:29:34.832956+0000","last_clean":"2024-09-21T10:29:34.832956+0000","last_became_active":"2024-09-21T10:25:48.765281+0000","last_became_peered":"2024-09-21T10:25:48.765281+0000","last_unstale":"2024-09-21T10:29:34.832956+0000","last_undegraded":"2024-09-21T10:29:34.832956+0000","last_fullsized":"2024-09-21T10:29:34.832956+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:41.798683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":455,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.830735+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:29:34.830735+0000","last_peered":"2024-09-21T10:29:34.830735+0000","last_clean":"2024-09-21T10:29:34.830735+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:29:34.830735+0000","last_undegraded":"2024-09-21T10:29:34.830735+0000","last_fullsized":"2024-09-21T10:29:34.830735+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":747,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827290+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:29:34.827290+0000","last_peered":"2024-09-21T10:29:34.827290+0000","last_clean":"2024-09-21T10:29:34.827290+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:29:34.827290+0000","last_undegraded":"2024-09-21T10:29:34.827290+0000","last_fullsized":"2024-09-21T10:29:34.827290+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":429,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750931+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:29:28.750931+0000","last_peered":"2024-09-21T10:29:28.750931+0000","last_clean":"2024-09-21T10:29:28.750931+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:29:28.750931+0000","last_undegraded":"2024-09-21T10:29:28.750931+0000","last_fullsized":"2024-09-21T10:29:28.750931+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":746,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.832525+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:29:34.832525+0000","last_peered":"2024-09-21T10:29:34.832525+0000","last_clean":"2024-09-21T10:29:34.832525+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:29:34.832525+0000","last_undegraded":"2024-09-21T10:29:34.832525+0000","last_fullsized":"2024-09-21T10:29:34.832525+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":93,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.836262+0000","last_change":"2024-09-21T10:25:48.767385+0000","last_active":"2024-09-21T10:29:34.836262+0000","last_peered":"2024-09-21T10:29:34.836262+0000","last_clean":"2024-09-21T10:29:34.836262+0000","last_became_active":"2024-09-21T10:25:48.764077+0000","last_became_peered":"2024-09-21T10:25:48.764077+0000","last_unstale":"2024-09-21T10:29:34.836262+0000","last_undegraded":"2024-09-21T10:29:34.836262+0000","last_fullsized":"2024-09-21T10:29:34.836262+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:06:51.969987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":121,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750331+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:29:28.750331+0000","last_peered":"2024-09-21T10:29:28.750331+0000","last_clean":"2024-09-21T10:29:28.750331+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:29:28.750331+0000","last_undegraded":"2024-09-21T10:29:28.750331+0000","last_fullsized":"2024-09-21T10:29:28.750331+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":365,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750181+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:29:28.750181+0000","last_peered":"2024-09-21T10:29:28.750181+0000","last_clean":"2024-09-21T10:29:28.750181+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:29:28.750181+0000","last_undegraded":"2024-09-21T10:29:28.750181+0000","last_fullsized":"2024-09-21T10:29:28.750181+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":136,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.750807+0000","last_change":"2024-09-21T10:25:48.753233+0000","last_active":"2024-09-21T10:29:28.750807+0000","last_peered":"2024-09-21T10:29:28.750807+0000","last_clean":"2024-09-21T10:29:28.750807+0000","last_became_active":"2024-09-21T10:25:48.752932+0000","last_became_peered":"2024-09-21T10:25:48.752932+0000","last_unstale":"2024-09-21T10:29:28.750807+0000","last_undegraded":"2024-09-21T10:29:28.750807+0000","last_fullsized":"2024-09-21T10:29:28.750807+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:49.456475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":343,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:28.751186+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:29:28.751186+0000","last_peered":"2024-09-21T10:29:28.751186+0000","last_clean":"2024-09-21T10:29:28.751186+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:29:28.751186+0000","last_undegraded":"2024-09-21T10:29:28.751186+0000","last_fullsized":"2024-09-21T10:29:28.751186+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":771,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827259+0000","last_change":"2024-09-21T10:25:48.755262+0000","last_active":"2024-09-21T10:29:34.827259+0000","last_peered":"2024-09-21T10:29:34.827259+0000","last_clean":"2024-09-21T10:29:34.827259+0000","last_became_active":"2024-09-21T10:25:48.754746+0000","last_became_peered":"2024-09-21T10:25:48.754746+0000","last_unstale":"2024-09-21T10:29:34.827259+0000","last_undegraded":"2024-09-21T10:29:34.827259+0000","last_fullsized":"2024-09-21T10:29:34.827259+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:28:17.312418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":722,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.827206+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:29:34.827206+0000","last_peered":"2024-09-21T10:29:34.827206+0000","last_clean":"2024-09-21T10:29:34.827206+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:29:34.827206+0000","last_undegraded":"2024-09-21T10:29:34.827206+0000","last_fullsized":"2024-09-21T10:29:34.827206+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":122,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.052454+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:29:34.052454+0000","last_peered":"2024-09-21T10:29:34.052454+0000","last_clean":"2024-09-21T10:29:34.052454+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:29:34.052454+0000","last_undegraded":"2024-09-21T10:29:34.052454+0000","last_fullsized":"2024-09-21T10:29:34.052454+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":125,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-21T10:29:34.833046+0000","last_change":"2024-09-21T10:25:48.764108+0000","last_active":"2024-09-21T10:29:34.833046+0000","last_peered":"2024-09-21T10:29:34.833046+0000","last_clean":"2024-09-21T10:29:34.833046+0000","last_became_active":"2024-09-21T10:25:48.761361+0000","last_became_peered":"2024-09-21T10:25:48.761361+0000","last_unstale":"2024-09-21T10:29:34.833046+0000","last_undegraded":"2024-09-21T10:29:34.833046+0000","last_fullsized":"2024-09-21T10:29:34.833046+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:49.188529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1654063104,"num_objects":25239,"num_object_clones":0,"num_object_copies":75717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":25239,"num_write_kb":1615296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5073403904,"data_stored":5073403904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":25239,"ondisk_log_size":25239,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28818,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234480,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205144,"kb_used_data":365968,"kb_used_omap":1,"kb_used_meta":839166,"kb_avail":92540008,"statfs":{"total":95995035648,"available":94760968192,"internally_reserved":0,"allocated":374751232,"data_stored":372599354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":859306443},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496118,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1384596,"kb_used_data":649612,"kb_used_omap":1,"kb_used_meta":734974,"kb_avail":92360556,"statfs":{"total":95995035648,"available":94577209344,"internally_reserved":0,"allocated":665202688,"data_stored":663055110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":752613831},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569404018,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1102352,"kb_used_data":547016,"kb_used_omap":7,"kb_used_meta":555320,"kb_avail":92642800,"statfs":{"total":95995035648,"available":94866227200,"internally_reserved":0,"allocated":560144384,"data_stored":557997272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":568648600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019391,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1418840,"kb_used_data":742096,"kb_used_omap":1,"kb_used_meta":676734,"kb_avail":92326312,"statfs":{"total":95995035648,"available":94542143488,"internally_reserved":0,"allocated":759906304,"data_stored":757754687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":692976070},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435575,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1495624,"kb_used_data":499072,"kb_used_omap":5,"kb_used_meta":996538,"kb_avail":92249528,"statfs":{"total":95995035648,"available":94463516672,"internally_reserved":0,"allocated":511049728,"data_stored":508910239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":1020454952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599102,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1403848,"kb_used_data":596096,"kb_used_omap":7,"kb_used_meta":807736,"kb_avail":92341304,"statfs":{"total":95995035648,"available":94557495296,"internally_reserved":0,"allocated":610402304,"data_stored":608263033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":827122125},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762629,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1162688,"kb_used_data":328568,"kb_used_omap":7,"kb_used_meta":834104,"kb_avail":92582464,"statfs":{"total":95995035648,"available":94804443136,"internally_reserved":0,"allocated":336453632,"data_stored":334321447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":854122749},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":389,"seq":1670742278192,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1296648,"kb_used_data":1146240,"kb_used_omap":12,"kb_used_meta":150387,"kb_avail":92448504,"statfs":{"total":95995035648,"available":94667268096,"internally_reserved":0,"allocated":1173749760,"data_stored":1171610241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12595,"internal_metadata":153997005},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11001,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1213464576,"data_stored":1213464576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":340262912,"data_stored":340262912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":616169472,"data_stored":616169472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":507183104,"data_stored":507183104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":768081920,"data_stored":768081920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":565248000,"data_stored":565248000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":669646848,"data_stored":669646848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":393347072,"data_stored":393347072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:29:47.542 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:29:47.543 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:29:47.543 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-21T10:29:47.543 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2024-09-21T10:29:47.543 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 6 1 2024-09-21T10:29:47.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:47.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:47.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:47.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:47.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:47.747 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:47.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:47.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:47.848 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:47.877 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:47.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:47.929 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:47 smithi032 ceph-mon[101299]: pgmap v1341: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-21T10:29:47.930 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:47 smithi032 ceph-mon[105748]: pgmap v1341: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-21T10:29:47.979 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:48.020 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:48.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:48.047 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:48.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:48.156 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:48.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:48.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:48.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 17 16 40581 40565 149.122 113.375 0.00459452 0.00670161 2024-09-21T10:29:48.234 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:48.257 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:48.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:47 smithi045 ceph-mon[95976]: pgmap v1341: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-21T10:29:48.298 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:48.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:48.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:48.400 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:48.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:48.441 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:48.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:48.541 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:48.588 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:48.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:48.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:48.690 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:48.727 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:48.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:48.828 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:48.856 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:29:48.869 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:48.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:48.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:48.888 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:48 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3523764224' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:29:48.888 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:48.888 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:48.971 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:49.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:49.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:49.121 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:49.131 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:29:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:29:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42102 10.0 e425/419 10:0891793a:::benchmark_data_smithi032_297285_object42101:head v 425'2617, mlcod=425'2610)", 2024-09-21T10:29:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.113388+0000", 2024-09-21T10:29:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.014076943, 2024-09-21T10:29:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014104719, 2024-09-21T10:29:49.131 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.113388+0000", 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.113388+0000", 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.113388+0000", 2024-09-21T10:29:49.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.113412+0000", 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4711000000000001e-05 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.113422+0000", 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.1919999999999994e-06 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.113427+0000", 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4489999999999998e-06 2024-09-21T10:29:49.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117141+0000", 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0037138549999999998 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117164+0000", 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2730000000000001e-05 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42108 10.2 e425/419 10:4b98f507:::benchmark_data_smithi032_297285_object42107:head v 425'2676, mlcod=425'2673)", 2024-09-21T10:29:49.134 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.117296+0000", 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.010169627000000001, 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010266673, 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117296+0000", 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117296+0000", 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117295+0000", 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117331+0000", 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5197000000000002e-05 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117338+0000", 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5390000000000002e-06 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117343+0000", 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2220000000000001e-06 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118949+0000", 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001605885 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118970+0000", 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0581000000000002e-05 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.137 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:42112 10.e 10:7c08f172:::benchmark_data_smithi032_297285_object42111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.117684+0000", 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0097808419999999997, 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0099459849999999992, 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 42112 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.138 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117684+0000", 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117684+0000", 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117683+0000", 2024-09-21T10:29:49.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117715+0000", 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2144000000000003e-05 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117728+0000", 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.273e-05 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117734+0000", 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3130000000000004e-06 2024-09-21T10:29:49.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.117870+0000", 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013618500000000001 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118005+0000", 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013445300000000001 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118146+0000", 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000141409 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123157+0000", 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0050103120000000003 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:42114 10.e 10:7c44ed78:::benchmark_data_smithi032_297285_object42113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.118649+0000", 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0088163719999999994, 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0090670439999999998, 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.142 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 42114 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118649+0000", 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118649+0000", 2024-09-21T10:29:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118648+0000", 2024-09-21T10:29:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:29:49.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118677+0000", 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9258999999999999e-05 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118693+0000", 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.573e-05 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.118701+0000", 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6429999999999995e-06 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.120907+0000", 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022058569999999999 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.121057+0000", 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015051499999999999 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.121249+0000", 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00019214300000000001 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123233+0000", 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0019841820000000001 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42111 10.2 e425/419 10:4f2d99ed:::benchmark_data_smithi032_297285_object42110:head v 425'2677, mlcod=425'2673)", 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.119334+0000", 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0081308839999999997, 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0084623770000000001, 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119334+0000", 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119334+0000", 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119334+0000", 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119503+0000", 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000169531 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119514+0000", 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1208e-05 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119521+0000", 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1670000000000004e-06 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.149 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.120350+0000", 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00082960399999999995 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.120367+0000", 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7028e-05 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.150 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42110 10.c e425/419 10:3c1f44b1:::benchmark_data_smithi032_297285_object42109:head v 425'2586, mlcod=425'2583)", 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.119546+0000", 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0079190519999999993, 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0083120429999999999, 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119546+0000", 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.151 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119546+0000", 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119546+0000", 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119565+0000", 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9287999999999999e-05 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.152 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119572+0000", 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2049999999999996e-06 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119577+0000", 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7480000000000001e-06 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119653+0000", 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6605000000000003e-05 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.153 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.119672+0000", 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8036000000000001e-05 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42118 10.c e425/419 10:37da465b:::benchmark_data_smithi032_297285_object42117:head v 425'2587, mlcod=425'2584)", 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.122523+0000", 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0049418769999999999, 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0053946150000000002, 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.154 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.122523+0000", 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.122523+0000", 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.122522+0000", 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.122547+0000", 2024-09-21T10:29:49.155 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4168999999999998e-05 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.122562+0000", 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5664000000000001e-05 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.122569+0000", 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4219999999999997e-06 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.122635+0000", 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6897999999999995e-05 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.122654+0000", 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8834e-05 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42119 10.2 e425/419 10:4c89a09b:::benchmark_data_smithi032_297285_object42118:head v 425'2678, mlcod=425'2673)", 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.123137+0000", 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0043280560000000003, 2024-09-21T10:29:49.157 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0048439570000000003, 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123137+0000", 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123137+0000", 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.158 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123136+0000", 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123216+0000", 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9548000000000005e-05 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123226+0000", 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0362e-05 2024-09-21T10:29:49.159 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123232+0000", 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9909999999999999e-06 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123284+0000", 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1180999999999997e-05 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.123301+0000", 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7580000000000001e-05 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.160 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42116 10.d e425/419 10:b29ffb52:::benchmark_data_smithi032_297285_object42115:head v 425'2638, mlcod=425'2636)", 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.125147+0000", 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00231796, 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0028930900000000001, 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.125147+0000", 2024-09-21T10:29:49.161 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.125147+0000", 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.125146+0000", 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.125173+0000", 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.7203999999999999e-05 2024-09-21T10:29:49.162 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.125189+0000", 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5994999999999998e-05 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.125197+0000", 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6680000000000001e-06 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.125264+0000", 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7281999999999997e-05 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.163 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.125290+0000", 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5795999999999998e-05 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42125 10.0 e425/419 10:0bc0aea6:::benchmark_data_smithi032_297285_object42124:head v 425'2618, mlcod=425'2615)", 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.126896+0000", 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00056975700000000005, 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0011990460000000001, 2024-09-21T10:29:49.164 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.126896+0000", 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.126896+0000", 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.126895+0000", 2024-09-21T10:29:49.165 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.126919+0000", 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4043e-05 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.126933+0000", 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4810999999999999e-05 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.126944+0000", 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0220999999999999e-05 2024-09-21T10:29:49.166 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.127035+0000", 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.1643000000000003e-05 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.127054+0000", 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8805e-05 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42120 10.c e425/419)", 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.127928+0000", 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4294967295.9995375, 2024-09-21T10:29:49.167 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00022086100000000001, 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.127928+0000", 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.127928+0000", 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.168 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.127927+0000", 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.127986+0000", 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8964e-05 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.127997+0000", 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1377e-05 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.128004+0000", 2024-09-21T10:29:49.169 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7689999999999999e-06 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:42124 10.2 e425/419)", 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:49.128024+0000", 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 4294967295.9994407, 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016912599999999999, 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.170 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.128024+0000", 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.128024+0000", 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.128024+0000", 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.171 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.128083+0000", 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8941999999999999e-05 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.128090+0000", 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3649999999999998e-06 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:49.128095+0000", 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7469999999999996e-06 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.172 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:29:49.173 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:29:49.173 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 12 2024-09-21T10:29:49.173 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:29:49.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:49.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:49.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:49.178 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 18 16 42189 42173 146.42 100.5 0.00516246 0.00682228 2024-09-21T10:29:49.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:48 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3523764224' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:29:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:48 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3523764224' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:29:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:29:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:29:49.275 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:49.317 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:49.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:49.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:49.444 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:49.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:49.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:49.546 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:49.585 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:49.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:49.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:49.687 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:49.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:49.725 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:49.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:49.826 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:49.847 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:49.848 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:29:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:29:49] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.51.0" 2024-09-21T10:29:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:49.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:49.949 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:49.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:49.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:50.090 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:50.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:49 smithi032 ceph-mon[101299]: pgmap v1342: 99 pgs: 99 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-21T10:29:50.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:49 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:29:50.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:49 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3523764224' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:29:50.116 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:49 smithi032 ceph-mon[101299]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T10:29:50.116 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:49 smithi032 ceph-mon[105748]: pgmap v1342: 99 pgs: 99 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-21T10:29:50.116 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:49 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:29:50.116 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:49 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3523764224' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:29:50.116 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:49 smithi032 ceph-mon[105748]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T10:29:50.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:50.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:50.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:50.134 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:49 smithi045 ceph-mon[95976]: pgmap v1342: 99 pgs: 99 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-21T10:29:50.134 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:49 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:29:50.134 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:49 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3523764224' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:29:50.134 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:49 smithi045 ceph-mon[95976]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T10:29:50.179 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 19 16 43739 43723 143.812 96.875 0.00841915 0.00694819 2024-09-21T10:29:50.230 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:50.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:50.271 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:50.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:50.372 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:50.411 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:50.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:50.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:50.513 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:50.551 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:50.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:50.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:50.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:50.692 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:50.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:50.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:50.794 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:50.834 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:50.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:50.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:50.972 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:50.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:50.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:51.074 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:51.112 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:51.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:51.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:51.179 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:29:51.177936+0000 min lat: 0.00102285 max lat: 0.278869 avg lat: 0.00707317 2024-09-21T10:29:51.179 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:29:51.179 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 20 16 45229 45213 141.277 93.125 0.0100932 0.00707317 2024-09-21T10:29:51.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:51.249 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:51.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:51.350 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:51.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:51.388 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:51.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:51.489 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:51.524 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:51.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:51.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:51.626 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:51.648 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:51.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:51.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:51.749 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:51.786 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:51.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:51.886 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:51.923 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:51.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:51.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:52.025 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:52.065 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:52.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:52.166 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:52.179 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 21 16 46711 46695 138.96 92.625 0.00557345 0.00719146 2024-09-21T10:29:52.199 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:52.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:52.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:51 smithi032 ceph-mon[101299]: pgmap v1344: 99 pgs: 99 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2024-09-21T10:29:52.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:51 smithi032 ceph-mon[105748]: pgmap v1344: 99 pgs: 99 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2024-09-21T10:29:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:51 smithi045 ceph-mon[95976]: pgmap v1344: 99 pgs: 99 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2024-09-21T10:29:52.300 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:52.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:52.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:52.437 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:52.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:52.477 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:52.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:52.578 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:52.618 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:52.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:52.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:52.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:29:52.719 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:52.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:52.771 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:52.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:52.872 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:52.898 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:52.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:52.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:52.998 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:29:53.000 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:53.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:53.043 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:53.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:53.144 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:53.179 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 22 16 48406 48390 137.457 105.938 0.00755259 0.00726929 2024-09-21T10:29:53.184 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:53.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:53.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:53.286 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:53.367 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:53.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:53.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:53.469 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:53.499 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:53.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:53.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:53.601 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:53.627 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:53.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:53.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:53.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:53.750 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:53.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:53.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:53.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:53.891 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:53.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:53.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:53.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:54.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:54.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:54.133 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:54.168 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:54.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:54.179 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 23 16 49704 49688 135.008 81.125 0.00643715 0.00740197 2024-09-21T10:29:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:53 smithi032 ceph-mon[101299]: pgmap v1345: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-21T10:29:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:53 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2484363155' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:29:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:53 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:29:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:53 smithi032 ceph-mon[105748]: pgmap v1345: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-21T10:29:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:53 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2484363155' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:29:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:53 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:29:54.217 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-21T10:29:54.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:53 smithi045 ceph-mon[95976]: pgmap v1345: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-21T10:29:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:53 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2484363155' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:29:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:53 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:29:54.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:54.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:54.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:54.411 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:54.446 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:54.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:54.512 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:29:54.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:54.584 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:54.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:54.685 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:54.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:54.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:54.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:54.842 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:54.877 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:54.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:54.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:54.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:55.019 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:55.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:55.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:55.121 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:55.154 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:55.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:55.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:55.182 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 24 16 51204 51188 133.275 93.75 0.0221377 0.00749651 2024-09-21T10:29:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:54 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:29:55.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:54 smithi032 ceph-mon[101299]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T10:29:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:54 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:29:55.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:54 smithi032 ceph-mon[105748]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T10:29:55.256 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:54 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:29:55.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:54 smithi045 ceph-mon[95976]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T10:29:55.304 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:55.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:55.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:55.405 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:29:55.448 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:55.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:29:55.550 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:29:55.559 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:29:55.559 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:29:55.587 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:55.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:55.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:29:55.598 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:29:55.599 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:29:55.638 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:29:55.639 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:29:55.675 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:29:55.675 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:29:55.689 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:55.708 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:29:55.709 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:29:55.727 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:55.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:55.735 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:29:55.735 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:29:55.783 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:29:55.783 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:29:55.816 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:29:55.817 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:29:55.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:55.863 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:55.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:55.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:55.965 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:56.003 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:56.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:56.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:56.049 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:29:56.049 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:29:56.077 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:55 smithi032 ceph-mon[105748]: pgmap v1347: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-21T10:29:56.077 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4162267807' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:56.077 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:55 smithi032 ceph-mon[101299]: pgmap v1347: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-21T10:29:56.077 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4162267807' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:56.105 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:56.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:56.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:56.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:56.182 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 25 16 52683 52667 131.641 92.4375 0.00671833 0.00759148 2024-09-21T10:29:56.250 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:56.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:55 smithi045 ceph-mon[95976]: pgmap v1347: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-21T10:29:56.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4162267807' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:29:56.286 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:56.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:56.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:56.388 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:56.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:56.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:56.532 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:56.569 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:56.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:56.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:56.670 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:56.705 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:56.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:56.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:56.807 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:56.849 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:56.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:56.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:56.914 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:29:56.950 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:56.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:56.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:56.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:57.090 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:57.126 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:57.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:57.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:57.182 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 26 16 54185 54169 130.189 93.875 0.0249563 0.00767545 2024-09-21T10:29:57.229 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:57.265 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:57.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:57.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:57.367 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:57.407 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:57.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:57.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:57.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:57.545 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:57.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:57.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:57.647 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:57.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:57.670 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:57.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:57.771 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:57.811 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:57.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:57.913 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:29:57.951 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:57.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:57.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:29:58.052 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:29:58.087 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:58.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:58.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:29:58.158 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:57 smithi045 ceph-mon[95976]: pgmap v1348: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T10:29:58.182 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 27 16 55595 55579 128.63 88.125 0.0100501 0.00776871 2024-09-21T10:29:58.189 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:58.197 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:29:58.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:57 smithi032 ceph-mon[101299]: pgmap v1348: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T10:29:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:57 smithi032 ceph-mon[105748]: pgmap v1348: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T10:29:58.227 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:58.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:58.328 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:58.370 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:58.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:58.471 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:58.518 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:58.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:58.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:58.599 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:29:58.620 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:58.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:58.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:58.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:58.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:58.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:58.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:58.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:58.887 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:29:58.913 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:58.956 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:58.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:59.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:29:59.100 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:29:59.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:59.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:29:59.182 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 28 16 56997 56981 127.166 87.625 0.00936444 0.00785876 2024-09-21T10:29:59.202 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:29:59.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:59.234 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:59.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:29:59.334 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:59.368 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:59.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:59.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:59.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:59.492 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:59.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:59.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:59.594 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:59.629 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:59.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:59.657 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:29:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:29:59] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.51.0" 2024-09-21T10:29:59.730 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:29:59.767 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:59.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:59.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:29:59.868 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:29:59.903 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:29:59.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:29:59.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:29:59.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:29:59 smithi032 ceph-mon[105748]: pgmap v1349: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-21T10:29:59.959 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:29:59 smithi032 ceph-mon[101299]: pgmap v1349: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-21T10:29:59.987 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:00.004 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:00.044 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:29:59 smithi045 ceph-mon[95976]: pgmap v1349: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-21T10:30:00.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:00.045 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:00.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:00.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:00.182 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 29 16 58442 58426 125.895 90.3125 0.011937 0.00793762 2024-09-21T10:30:00.192 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:00.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:00.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:00.294 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:00.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:00.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:00.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:00.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:00.461 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:00.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:00.563 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:00.597 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:00.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:00.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:00.699 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:00.735 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:00.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:00.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:00.837 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:00.885 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:00.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:00.905 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:30:00.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:01.022 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:01.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:01.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:01.030 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:00 smithi045 ceph-mon[95976]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-21T10:30:01.030 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:00 smithi045 ceph-mon[95976]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-21T10:30:01.030 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:00 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1343612181' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:01.030 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:01.118 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 1, 0, 4] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 6, 7, 5, 0] 2024-09-21T10:30:01.118 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:30:01.118 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-21T10:30:01.118 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:30:01.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:01.158 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:00 smithi032 ceph-mon[101299]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-21T10:30:01.158 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:00 smithi032 ceph-mon[101299]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-21T10:30:01.158 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:00 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1343612181' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:01.158 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:01.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:00 smithi032 ceph-mon[105748]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-21T10:30:01.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:00 smithi032 ceph-mon[105748]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-21T10:30:01.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:00 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1343612181' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:01.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:01.184 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 30 16 59928 59912 124.788 92.875 0.0129356 0.00800832 2024-09-21T10:30:01.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:01.209 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:01.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:01.277 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:30:01.277 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:30:01.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:01.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:01.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:01.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:01.449 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:01.511 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:01.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:01.612 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:01.655 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:01.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:01.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:01.757 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:01.794 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:01.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:01.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:01.895 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:01.932 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:01.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:02.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:01 smithi045 ceph-mon[95976]: pgmap v1350: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-09-21T10:30:02.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:01 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:02.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:01 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:30:02.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:01 smithi045 ceph-mon[95976]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T10:30:02.034 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:02.063 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:30:02.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:02.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:02.174 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:02.184 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 31 16 61388 61372 123.706 91.25 0.00866915 0.00807794 2024-09-21T10:30:02.192 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:01 smithi032 ceph-mon[101299]: pgmap v1350: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-09-21T10:30:02.192 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:01 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:02.192 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:01 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:30:02.192 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:01 smithi032 ceph-mon[101299]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T10:30:02.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:01 smithi032 ceph-mon[105748]: pgmap v1350: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-09-21T10:30:02.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:01 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:02.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:01 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:30:02.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:01 smithi032 ceph-mon[105748]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T10:30:02.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:02.217 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:02.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:02.317 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:02.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:02.370 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:02.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:02.404 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:30:02.405 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:30:02.405 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:02.433 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:30:02.433 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:30:02.463 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:30:02.463 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:30:02.471 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:02.496 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:30:02.496 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:30:02.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:02.511 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:02.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:02.529 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:30:02.529 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:30:02.563 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:30:02.564 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:30:02.600 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:30:02.600 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:30:02.612 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:02.642 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:30:02.642 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:30:02.648 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:02.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:02.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:02.750 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:02.790 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:02.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:02.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:02.930 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:02.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:02.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:03.031 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:03.074 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:03.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:03.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:03.176 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:03.184 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 32 16 62860 62844 122.714 92 0.0122857 0.00814373 2024-09-21T10:30:03.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:02 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1827884669' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:02 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1827884669' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:03.212 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:03.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:03.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:02 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1827884669' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:03.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:03.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:03.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:03.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:03.455 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:03.498 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:03.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:03.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:03.600 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:03.640 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:03.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:03.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:03.742 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:03.783 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:03.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:03.884 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:03.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:03.944 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:03.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:04.045 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:04.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:04.088 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:04.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:04.164 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:03 smithi045 ceph-mon[95976]: pgmap v1352: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-21T10:30:04.164 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:04.184 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 33 15 64403 64388 121.92 96.5 0.00356666 0.00819617 2024-09-21T10:30:04.189 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:04.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:03 smithi032 ceph-mon[101299]: pgmap v1352: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-21T10:30:04.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:04.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:03 smithi032 ceph-mon[105748]: pgmap v1352: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-21T10:30:04.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:04.214 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:04.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:04.316 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:04.349 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:04.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:04.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:04.451 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:04.494 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:04.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:04.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:04.596 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:04.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:04.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:04.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:04.757 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:04.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:04.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:04.859 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:04.897 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:04.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:04.999 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:05.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:05.035 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:05.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:05.068 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:30:05.136 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:05.185 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:05.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:05.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 34 16 65761 65745 120.827 84.8125 0.00759051 0.00827101 2024-09-21T10:30:05.286 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:05.317 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:30:05.317 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:30:05.317 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.317 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:64774 10.0 10:012ef6a2:::benchmark_data_smithi032_297285_object64773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:30:05.317 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.471260+0000", 2024-09-21T10:30:05.317 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.01216392, 2024-09-21T10:30:05.317 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.012198583000000001, 2024-09-21T10:30:05.318 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.318 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:05.318 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:05.318 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:05.318 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:05.318 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 64774 2024-09-21T10:30:05.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.319 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.319 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.319 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471260+0000", 2024-09-21T10:30:05.319 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.319 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.319 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471260+0000", 2024-09-21T10:30:05.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.320 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.320 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.320 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.320 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471260+0000", 2024-09-21T10:30:05.320 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:05.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.321 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471354+0000", 2024-09-21T10:30:05.321 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4110999999999994e-05 2024-09-21T10:30:05.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.321 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.321 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.322 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471363+0000", 2024-09-21T10:30:05.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4669999999999993e-06 2024-09-21T10:30:05.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.322 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.322 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.322 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471369+0000", 2024-09-21T10:30:05.322 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3049999999999998e-06 2024-09-21T10:30:05.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:05.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471455+0000", 2024-09-21T10:30:05.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5234000000000004e-05 2024-09-21T10:30:05.323 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.323 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.323 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:05.323 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471618+0000", 2024-09-21T10:30:05.323 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000163387 2024-09-21T10:30:05.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.324 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:30:05.324 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471771+0000", 2024-09-21T10:30:05.324 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015263899999999999 2024-09-21T10:30:05.324 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.324 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.324 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:05.324 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478614+0000", 2024-09-21T10:30:05.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0068437250000000002 2024-09-21T10:30:05.325 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.325 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.325 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:05.325 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478623+0000", 2024-09-21T10:30:05.325 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1000000000000004e-06 2024-09-21T10:30:05.325 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.325 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.325 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.326 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.326 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:64775 10.0 10:018e4ef1:::benchmark_data_smithi032_297285_object64774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:30:05.326 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.472520+0000", 2024-09-21T10:30:05.326 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.010904518, 2024-09-21T10:30:05.326 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011045619, 2024-09-21T10:30:05.326 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.327 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:05.327 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:05.327 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:05.327 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:05.327 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 64775 2024-09-21T10:30:05.327 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.327 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.327 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.327 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.328 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.472520+0000", 2024-09-21T10:30:05.328 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.328 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.328 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.328 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.472520+0000", 2024-09-21T10:30:05.328 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.329 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.472519+0000", 2024-09-21T10:30:05.329 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:05.329 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.329 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.329 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.329 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.472550+0000", 2024-09-21T10:30:05.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0995999999999999e-05 2024-09-21T10:30:05.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.330 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.330 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.472560+0000", 2024-09-21T10:30:05.330 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0276e-05 2024-09-21T10:30:05.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.330 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.472566+0000", 2024-09-21T10:30:05.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.207e-06 2024-09-21T10:30:05.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.331 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.331 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:05.331 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.474196+0000", 2024-09-21T10:30:05.331 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001629779 2024-09-21T10:30:05.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:05.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.474357+0000", 2024-09-21T10:30:05.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000160944 2024-09-21T10:30:05.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.332 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.332 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:30:05.332 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.474507+0000", 2024-09-21T10:30:05.332 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000150171 2024-09-21T10:30:05.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:05.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478659+0000", 2024-09-21T10:30:05.333 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0041513950000000004 2024-09-21T10:30:05.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.333 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.333 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:05.333 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481790+0000", 2024-09-21T10:30:05.334 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003131726 2024-09-21T10:30:05.334 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.334 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.334 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.334 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.334 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:64778 10.a 10:5bd82ff0:::benchmark_data_smithi032_297285_object64777:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:30:05.334 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.473450+0000", 2024-09-21T10:30:05.335 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0099745159999999992, 2024-09-21T10:30:05.335 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010209031, 2024-09-21T10:30:05.335 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.335 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:05.335 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:05.335 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:05.335 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:05.335 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 64778 2024-09-21T10:30:05.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.336 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.336 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.336 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.473450+0000", 2024-09-21T10:30:05.336 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.336 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.336 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.473450+0000", 2024-09-21T10:30:05.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.337 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.337 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.337 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.473449+0000", 2024-09-21T10:30:05.337 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:05.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.473524+0000", 2024-09-21T10:30:05.338 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4725e-05 2024-09-21T10:30:05.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.338 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.338 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.338 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.473539+0000", 2024-09-21T10:30:05.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5620999999999998e-05 2024-09-21T10:30:05.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.339 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.339 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.473546+0000", 2024-09-21T10:30:05.339 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0809999999999996e-06 2024-09-21T10:30:05.339 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.339 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:05.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.473587+0000", 2024-09-21T10:30:05.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0991000000000001e-05 2024-09-21T10:30:05.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.340 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.340 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:05.340 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.473726+0000", 2024-09-21T10:30:05.340 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013851 2024-09-21T10:30:05.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.341 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.341 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:30:05.341 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.473858+0000", 2024-09-21T10:30:05.341 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013232599999999999 2024-09-21T10:30:05.341 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.341 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.341 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:05.341 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.476559+0000", 2024-09-21T10:30:05.342 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0027008560000000002 2024-09-21T10:30:05.342 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.342 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.342 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:05.342 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481721+0000", 2024-09-21T10:30:05.342 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0051615619999999997 2024-09-21T10:30:05.342 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.342 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.343 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.343 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.343 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:64782 10.7 10:edf3527c:::benchmark_data_smithi032_297285_object64781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:30:05.343 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.478811+0000", 2024-09-21T10:30:05.343 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0046135689999999997, 2024-09-21T10:30:05.343 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0049385480000000001, 2024-09-21T10:30:05.343 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.343 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:05.344 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:05.344 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:05.344 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:05.344 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 64782 2024-09-21T10:30:05.344 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.344 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.344 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.344 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478811+0000", 2024-09-21T10:30:05.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.345 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.345 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478811+0000", 2024-09-21T10:30:05.345 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.345 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.345 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.346 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478809+0000", 2024-09-21T10:30:05.346 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:05.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.346 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.346 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.346 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478850+0000", 2024-09-21T10:30:05.346 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1088999999999999e-05 2024-09-21T10:30:05.347 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.347 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.347 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.347 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478879+0000", 2024-09-21T10:30:05.347 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8510999999999999e-05 2024-09-21T10:30:05.347 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.347 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.348 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.348 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478887+0000", 2024-09-21T10:30:05.348 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2239999999999995e-06 2024-09-21T10:30:05.348 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.348 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.348 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:05.348 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.478938+0000", 2024-09-21T10:30:05.348 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0936000000000002e-05 2024-09-21T10:30:05.348 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.349 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.349 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:05.349 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.479102+0000", 2024-09-21T10:30:05.349 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016443499999999999 2024-09-21T10:30:05.349 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.349 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.349 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:30:05.349 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.479254+0000", 2024-09-21T10:30:05.349 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015154 2024-09-21T10:30:05.350 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.350 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.350 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:05.350 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481837+0000", 2024-09-21T10:30:05.350 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0025832350000000001 2024-09-21T10:30:05.350 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.350 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.351 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:05.351 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482442+0000", 2024-09-21T10:30:05.351 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000605264 2024-09-21T10:30:05.351 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.351 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.351 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.351 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.351 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.351 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:64785 10.a 10:5904b72c:::benchmark_data_smithi032_297285_object64784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:30:05.352 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.481399+0000", 2024-09-21T10:30:05.352 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0020253290000000002, 2024-09-21T10:30:05.352 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0024391949999999999, 2024-09-21T10:30:05.352 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.352 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:05.352 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:05.352 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:05.352 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:05.352 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 64785 2024-09-21T10:30:05.353 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.353 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.353 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.353 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.353 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481399+0000", 2024-09-21T10:30:05.353 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.353 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.354 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.354 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.354 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481399+0000", 2024-09-21T10:30:05.354 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.354 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.354 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.354 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.354 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481397+0000", 2024-09-21T10:30:05.355 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:30:05.355 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.355 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.355 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.355 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481432+0000", 2024-09-21T10:30:05.355 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5145000000000003e-05 2024-09-21T10:30:05.356 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.356 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.356 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.356 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481452+0000", 2024-09-21T10:30:05.356 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9698999999999999e-05 2024-09-21T10:30:05.356 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.356 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.356 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.357 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481460+0000", 2024-09-21T10:30:05.357 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4120000000000001e-06 2024-09-21T10:30:05.357 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.357 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.357 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:05.357 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481746+0000", 2024-09-21T10:30:05.357 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00028588799999999998 2024-09-21T10:30:05.358 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.358 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.358 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:05.358 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481869+0000", 2024-09-21T10:30:05.358 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012259700000000001 2024-09-21T10:30:05.358 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.358 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.358 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:30:05.359 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.481960+0000", 2024-09-21T10:30:05.359 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0620999999999999e-05 2024-09-21T10:30:05.359 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.359 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.359 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.359 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.360 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.36234.0:64770 10.a e427/419)", 2024-09-21T10:30:05.360 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.482565+0000", 2024-09-21T10:30:05.360 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00085981000000000002, 2024-09-21T10:30:05.360 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00133683, 2024-09-21T10:30:05.360 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.360 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:30:05.360 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.360 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.360 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.361 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482565+0000", 2024-09-21T10:30:05.361 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.361 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.361 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.361 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.361 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482565+0000", 2024-09-21T10:30:05.361 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.361 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.362 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.362 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.362 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482564+0000", 2024-09-21T10:30:05.362 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:05.362 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.362 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.362 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.362 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482569+0000", 2024-09-21T10:30:05.363 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.011e-06 2024-09-21T10:30:05.363 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.363 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.363 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.363 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482581+0000", 2024-09-21T10:30:05.363 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2601e-05 2024-09-21T10:30:05.363 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.363 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.364 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.364 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482587+0000", 2024-09-21T10:30:05.364 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6470000000000003e-06 2024-09-21T10:30:05.364 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.364 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.364 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.364 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.364 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.365 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.36234.0:64778 10.a e427/419)", 2024-09-21T10:30:05.365 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.482601+0000", 2024-09-21T10:30:05.365 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00082317999999999996, 2024-09-21T10:30:05.365 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013431319999999999, 2024-09-21T10:30:05.365 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.365 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:30:05.365 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.366 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.366 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.366 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482601+0000", 2024-09-21T10:30:05.366 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.366 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.366 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.366 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.366 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482601+0000", 2024-09-21T10:30:05.367 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.367 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.367 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.367 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.367 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482601+0000", 2024-09-21T10:30:05.367 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:05.367 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.367 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.368 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.368 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482603+0000", 2024-09-21T10:30:05.368 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.137e-06 2024-09-21T10:30:05.368 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.368 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.368 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.368 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482608+0000", 2024-09-21T10:30:05.368 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3149999999999999e-06 2024-09-21T10:30:05.369 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.369 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.369 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.369 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482612+0000", 2024-09-21T10:30:05.369 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.817e-06 2024-09-21T10:30:05.369 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.369 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.369 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.370 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.370 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.370 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:64786 10.d 10:b485ac13:::benchmark_data_smithi032_297285_object64785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:30:05.370 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.482629+0000", 2024-09-21T10:30:05.370 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.00079545099999999997, 2024-09-21T10:30:05.370 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013630700000000001, 2024-09-21T10:30:05.370 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.370 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:05.371 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:05.371 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:05.371 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:05.371 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 64786 2024-09-21T10:30:05.371 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.371 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.371 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.371 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482629+0000", 2024-09-21T10:30:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482629+0000", 2024-09-21T10:30:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.373 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.373 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.373 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482628+0000", 2024-09-21T10:30:05.373 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:05.373 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.373 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.373 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.374 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482657+0000", 2024-09-21T10:30:05.374 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9254000000000001e-05 2024-09-21T10:30:05.374 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.374 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.374 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.374 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482669+0000", 2024-09-21T10:30:05.374 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1593999999999999e-05 2024-09-21T10:30:05.374 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.375 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.375 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.375 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482674+0000", 2024-09-21T10:30:05.375 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4759999999999997e-06 2024-09-21T10:30:05.375 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.375 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.375 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:05.375 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482736+0000", 2024-09-21T10:30:05.376 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1679000000000004e-05 2024-09-21T10:30:05.376 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.376 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.376 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:05.376 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482864+0000", 2024-09-21T10:30:05.376 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012805999999999999 2024-09-21T10:30:05.376 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.376 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.376 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:30:05.377 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.482996+0000", 2024-09-21T10:30:05.377 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013156299999999999 2024-09-21T10:30:05.377 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.377 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.377 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.377 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.377 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.377 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:64762 10.0 10:044befe3:::benchmark_data_smithi032_297285_object64761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:30:05.378 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.458215+0000", 2024-09-21T10:30:05.378 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.025209334999999999, 2024-09-21T10:30:05.378 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.025846836000000002, 2024-09-21T10:30:05.378 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.378 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:05.378 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:05.378 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:05.378 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:05.379 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 64762 2024-09-21T10:30:05.379 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.379 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.379 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.379 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.379 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.458215+0000", 2024-09-21T10:30:05.379 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.380 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.380 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.380 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.380 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.458215+0000", 2024-09-21T10:30:05.380 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.380 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.380 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.380 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.381 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.458214+0000", 2024-09-21T10:30:05.381 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:05.381 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.381 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.381 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.381 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.458244+0000", 2024-09-21T10:30:05.381 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0232000000000001e-05 2024-09-21T10:30:05.381 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.382 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.382 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.382 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.458255+0000", 2024-09-21T10:30:05.382 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0594999999999999e-05 2024-09-21T10:30:05.382 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.382 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.382 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.383 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.458261+0000", 2024-09-21T10:30:05.383 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4840000000000001e-06 2024-09-21T10:30:05.383 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.383 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.383 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:05.383 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.464001+0000", 2024-09-21T10:30:05.383 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0057393339999999996 2024-09-21T10:30:05.383 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.384 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.384 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:05.384 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.464167+0000", 2024-09-21T10:30:05.384 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016627600000000001 2024-09-21T10:30:05.384 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.384 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.384 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:30:05.384 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.464321+0000", 2024-09-21T10:30:05.384 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00015405700000000001 2024-09-21T10:30:05.385 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.385 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.385 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:05.385 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.467048+0000", 2024-09-21T10:30:05.385 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0027269400000000002 2024-09-21T10:30:05.385 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.385 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.386 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:05.386 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.469632+0000", 2024-09-21T10:30:05.386 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002584425 2024-09-21T10:30:05.386 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.386 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.386 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.386 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.386 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.387 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:64770 10.a 10:5e7facc2:::benchmark_data_smithi032_297285_object64769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:30:05.387 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:04.468532+0000", 2024-09-21T10:30:05.387 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.014892166, 2024-09-21T10:30:05.387 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015610927, 2024-09-21T10:30:05.387 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:05.387 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:05.387 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:05.388 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:05.388 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:05.388 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 64770 2024-09-21T10:30:05.388 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.388 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:05.388 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.388 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:05.388 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.468532+0000", 2024-09-21T10:30:05.389 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.389 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.389 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.389 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:05.389 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.468532+0000", 2024-09-21T10:30:05.389 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:05.389 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.389 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.390 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:05.390 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.468531+0000", 2024-09-21T10:30:05.390 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:05.390 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.390 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.390 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:05.390 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.468568+0000", 2024-09-21T10:30:05.390 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6729e-05 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.468579+0000", 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1054e-05 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.468584+0000", 2024-09-21T10:30:05.391 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1359999999999996e-06 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.468617+0000", 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2691000000000003e-05 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.468702+0000", 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5212999999999999e-05 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.468780+0000", 2024-09-21T10:30:05.392 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8533000000000002e-05 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.471935+0000", 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031546809999999999 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:04.472355+0000", 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00041964599999999997 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:30:05.393 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 10 2024-09-21T10:30:05.394 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:30:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:05.395 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:05.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:05.496 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:05.522 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:05.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:05.623 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:05.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:05.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:05.759 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:05.796 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:05.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:05.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:05.898 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:05.938 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:05.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:05.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:06.040 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:06.076 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:06.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:06.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:06.177 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:06.184 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 35 16 67005 66989 119.597 77.75 0.00274279 0.00833967 2024-09-21T10:30:06.199 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:06.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:06.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:05 smithi032 ceph-mon[101299]: pgmap v1353: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-21T10:30:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:05 smithi032 ceph-mon[105748]: pgmap v1353: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-21T10:30:06.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:30:06.264 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:06.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:05 smithi045 ceph-mon[95976]: pgmap v1353: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-21T10:30:06.301 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:06.339 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:06.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:06.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:06.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:06.504 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:06.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:06.575 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:30:06.576 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:30:06.605 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:06.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:06.647 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:06.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:06.748 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:06.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:06.770 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:06.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:06.871 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:06.912 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:06.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:07.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:07.048 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:07.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:07.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:07.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:07.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 36 16 68080 68064 118.141 67.1875 0.00732954 0.00845971 2024-09-21T10:30:07.187 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:07.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:07.288 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:07.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:07.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:07.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:07.422 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:07.461 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:07.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:07.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:07.563 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:07.601 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:07.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:07.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:07.703 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:07.740 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:07.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:07.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:07.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:07.878 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:07.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:07.980 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:08.021 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:08.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:08.123 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:08.124 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:07 smithi045 ceph-mon[95976]: from='client.36273 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:30:08.124 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:07 smithi045 ceph-mon[95976]: pgmap v1354: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:30:08.156 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:08.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:08.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:08.164 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:30:08.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 37 16 69403 69387 117.183 82.6875 0.0158177 0.00852845 2024-09-21T10:30:08.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:07 smithi032 ceph-mon[101299]: from='client.36273 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:30:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:07 smithi032 ceph-mon[101299]: pgmap v1354: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:30:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:07 smithi032 ceph-mon[105748]: from='client.36273 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:30:08.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:07 smithi032 ceph-mon[105748]: pgmap v1354: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:30:08.258 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:08.292 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:08.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:08.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:08.357 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:08.365 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1353,"stamp":"2024-09-21T10:30:04.706943+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4148238309,"num_objects":63252,"num_object_clones":0,"num_object_copies":189756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63252,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":64132,"num_write_kb":4058961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":285,"num_bytes_recovered":16782981,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":65898,"ondisk_log_size":65898,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17719340,"kb_used_data":11910376,"kb_used_omap":45,"kb_used_meta":5808850,"kb_avail":732241876,"statfs":{"total":767960285184,"available":749815681024,"internally_reserved":0,"allocated":12196225024,"data_stored":12178992460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46417,"internal_metadata":5948263087},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":965672960,"num_objects":14735,"num_object_clones":0,"num_object_copies":44205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14735,"num_write_kb":943040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003269"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":732,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.863704+0000","last_change":"2024-09-21T10:13:09.578194+0000","last_active":"2024-09-21T10:29:53.863704+0000","last_peered":"2024-09-21T10:29:53.863704+0000","last_clean":"2024-09-21T10:29:53.863704+0000","last_became_active":"2024-09-21T10:13:09.577794+0000","last_became_peered":"2024-09-21T10:13:09.577794+0000","last_unstale":"2024-09-21T10:29:53.863704+0000","last_undegraded":"2024-09-21T10:29:53.863704+0000","last_fullsized":"2024-09-21T10:29:53.863704+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:05.104921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":433,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.901782+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:30:00.901782+0000","last_peered":"2024-09-21T10:30:00.901782+0000","last_clean":"2024-09-21T10:30:00.901782+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:30:00.901782+0000","last_undegraded":"2024-09-21T10:30:00.901782+0000","last_fullsized":"2024-09-21T10:30:00.901782+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":129,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.864359+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:29:53.864359+0000","last_peered":"2024-09-21T10:29:53.864359+0000","last_clean":"2024-09-21T10:29:53.864359+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:29:53.864359+0000","last_undegraded":"2024-09-21T10:29:53.864359+0000","last_fullsized":"2024-09-21T10:29:53.864359+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":100,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.934911+0000","last_change":"2024-09-21T10:25:48.765404+0000","last_active":"2024-09-21T10:30:00.934911+0000","last_peered":"2024-09-21T10:30:00.934911+0000","last_clean":"2024-09-21T10:30:00.934911+0000","last_became_active":"2024-09-21T10:25:48.764230+0000","last_became_peered":"2024-09-21T10:25:48.764230+0000","last_unstale":"2024-09-21T10:30:00.934911+0000","last_undegraded":"2024-09-21T10:30:00.934911+0000","last_fullsized":"2024-09-21T10:30:00.934911+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:38.551941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":101,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.934037+0000","last_change":"2024-09-21T10:25:48.764391+0000","last_active":"2024-09-21T10:30:00.934037+0000","last_peered":"2024-09-21T10:30:00.934037+0000","last_clean":"2024-09-21T10:30:00.934037+0000","last_became_active":"2024-09-21T10:25:48.763375+0000","last_became_peered":"2024-09-21T10:25:48.763375+0000","last_unstale":"2024-09-21T10:30:00.934037+0000","last_undegraded":"2024-09-21T10:30:00.934037+0000","last_fullsized":"2024-09-21T10:30:00.934037+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:03.457716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":404,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.901774+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:30:00.901774+0000","last_peered":"2024-09-21T10:30:00.901774+0000","last_clean":"2024-09-21T10:30:00.901774+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:30:00.901774+0000","last_undegraded":"2024-09-21T10:30:00.901774+0000","last_fullsized":"2024-09-21T10:30:00.901774+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":752,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.862304+0000","last_change":"2024-09-21T10:21:34.866038+0000","last_active":"2024-09-21T10:29:53.862304+0000","last_peered":"2024-09-21T10:29:53.862304+0000","last_clean":"2024-09-21T10:29:53.862304+0000","last_became_active":"2024-09-21T10:21:34.865711+0000","last_became_peered":"2024-09-21T10:21:34.865711+0000","last_unstale":"2024-09-21T10:29:53.862304+0000","last_undegraded":"2024-09-21T10:29:53.862304+0000","last_fullsized":"2024-09-21T10:29:53.862304+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:20.088795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":500,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:59.167512+0000","last_change":"2024-09-21T10:25:48.752712+0000","last_active":"2024-09-21T10:29:59.167512+0000","last_peered":"2024-09-21T10:29:59.167512+0000","last_clean":"2024-09-21T10:29:59.167512+0000","last_became_active":"2024-09-21T10:25:48.752204+0000","last_became_peered":"2024-09-21T10:25:48.752204+0000","last_unstale":"2024-09-21T10:29:59.167512+0000","last_undegraded":"2024-09-21T10:29:59.167512+0000","last_fullsized":"2024-09-21T10:29:59.167512+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:40.068520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"427'3715","reported_seq":3732,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:01.053925+0000","last_change":"2024-09-21T10:29:18.322432+0000","last_active":"2024-09-21T10:30:01.053925+0000","last_peered":"2024-09-21T10:30:01.053925+0000","last_clean":"2024-09-21T10:30:01.053925+0000","last_became_active":"2024-09-21T10:29:18.322116+0000","last_became_peered":"2024-09-21T10:29:18.322116+0000","last_unstale":"2024-09-21T10:30:01.053925+0000","last_undegraded":"2024-09-21T10:30:01.053925+0000","last_fullsized":"2024-09-21T10:30:01.053925+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3715,"log_dups_size":0,"ondisk_log_size":3715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:35.158580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243466240,"num_objects":3715,"num_object_clones":0,"num_object_copies":11145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3715,"num_write_kb":237760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":134,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.862231+0000","last_change":"2024-09-21T10:25:48.755730+0000","last_active":"2024-09-21T10:29:53.862231+0000","last_peered":"2024-09-21T10:29:53.862231+0000","last_clean":"2024-09-21T10:29:53.862231+0000","last_became_active":"2024-09-21T10:25:48.755344+0000","last_became_peered":"2024-09-21T10:25:48.755344+0000","last_unstale":"2024-09-21T10:29:53.862231+0000","last_undegraded":"2024-09-21T10:29:53.862231+0000","last_fullsized":"2024-09-21T10:29:53.862231+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:37.725090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":137,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.863972+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:29:53.863972+0000","last_peered":"2024-09-21T10:29:53.863972+0000","last_clean":"2024-09-21T10:29:53.863972+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:29:53.863972+0000","last_undegraded":"2024-09-21T10:29:53.863972+0000","last_fullsized":"2024-09-21T10:29:53.863972+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"427'3950","reported_seq":3967,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.178359+0000","last_change":"2024-09-21T10:29:18.320824+0000","last_active":"2024-09-21T10:30:04.178359+0000","last_peered":"2024-09-21T10:30:04.178359+0000","last_clean":"2024-09-21T10:30:04.178359+0000","last_became_active":"2024-09-21T10:29:18.320520+0000","last_became_peered":"2024-09-21T10:29:18.320520+0000","last_unstale":"2024-09-21T10:30:04.178359+0000","last_undegraded":"2024-09-21T10:30:04.178359+0000","last_fullsized":"2024-09-21T10:30:04.178359+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3950,"log_dups_size":0,"ondisk_log_size":3950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:23.749267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258867200,"num_objects":3950,"num_object_clones":0,"num_object_copies":11850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3950,"num_write_kb":252800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":106,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.862050+0000","last_change":"2024-09-21T10:21:34.870569+0000","last_active":"2024-09-21T10:29:53.862050+0000","last_peered":"2024-09-21T10:29:53.862050+0000","last_clean":"2024-09-21T10:29:53.862050+0000","last_became_active":"2024-09-21T10:21:34.862094+0000","last_became_peered":"2024-09-21T10:21:34.862094+0000","last_unstale":"2024-09-21T10:29:53.862050+0000","last_undegraded":"2024-09-21T10:29:53.862050+0000","last_fullsized":"2024-09-21T10:29:53.862050+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:39:44.848393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":125,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.865754+0000","last_change":"2024-09-21T10:19:50.584073+0000","last_active":"2024-09-21T10:29:53.865754+0000","last_peered":"2024-09-21T10:29:53.865754+0000","last_clean":"2024-09-21T10:29:53.865754+0000","last_became_active":"2024-09-21T10:19:50.583745+0000","last_became_peered":"2024-09-21T10:19:50.583745+0000","last_unstale":"2024-09-21T10:29:53.865754+0000","last_undegraded":"2024-09-21T10:29:53.865754+0000","last_fullsized":"2024-09-21T10:29:53.865754+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:56.185518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"427'3871","reported_seq":3898,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:02.184705+0000","last_change":"2024-09-21T10:29:18.320476+0000","last_active":"2024-09-21T10:30:02.184705+0000","last_peered":"2024-09-21T10:30:02.184705+0000","last_clean":"2024-09-21T10:30:02.184705+0000","last_became_active":"2024-09-21T10:29:18.320167+0000","last_became_peered":"2024-09-21T10:29:18.320167+0000","last_unstale":"2024-09-21T10:30:02.184705+0000","last_undegraded":"2024-09-21T10:30:02.184705+0000","last_fullsized":"2024-09-21T10:30:02.184705+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3871,"log_dups_size":0,"ondisk_log_size":3871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:38:15.611579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253689856,"num_objects":3871,"num_object_clones":0,"num_object_copies":11613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3871,"num_write_kb":247744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":428,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861597+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:29:53.861597+0000","last_peered":"2024-09-21T10:29:53.861597+0000","last_clean":"2024-09-21T10:29:53.861597+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:29:53.861597+0000","last_undegraded":"2024-09-21T10:29:53.861597+0000","last_fullsized":"2024-09-21T10:29:53.861597+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":448,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.872015+0000","last_change":"2024-09-21T10:21:34.864991+0000","last_active":"2024-09-21T10:29:53.872015+0000","last_peered":"2024-09-21T10:29:53.872015+0000","last_clean":"2024-09-21T10:29:53.872015+0000","last_became_active":"2024-09-21T10:21:34.864693+0000","last_became_peered":"2024-09-21T10:21:34.864693+0000","last_unstale":"2024-09-21T10:29:53.872015+0000","last_undegraded":"2024-09-21T10:29:53.872015+0000","last_fullsized":"2024-09-21T10:29:53.872015+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:59.406905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"427'3992","reported_seq":4019,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.695026+0000","last_change":"2024-09-21T10:29:18.321766+0000","last_active":"2024-09-21T10:30:04.695026+0000","last_peered":"2024-09-21T10:30:04.695026+0000","last_clean":"2024-09-21T10:30:04.695026+0000","last_became_active":"2024-09-21T10:29:18.321306+0000","last_became_peered":"2024-09-21T10:29:18.321306+0000","last_unstale":"2024-09-21T10:30:04.695026+0000","last_undegraded":"2024-09-21T10:30:04.695026+0000","last_fullsized":"2024-09-21T10:30:04.695026+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3992,"log_dups_size":0,"ondisk_log_size":3992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:39:43.157433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261619712,"num_objects":3992,"num_object_clones":0,"num_object_copies":11976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3992,"num_write_kb":255488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":362,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.864338+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:29:53.864338+0000","last_peered":"2024-09-21T10:29:53.864338+0000","last_clean":"2024-09-21T10:29:53.864338+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:29:53.864338+0000","last_undegraded":"2024-09-21T10:29:53.864338+0000","last_fullsized":"2024-09-21T10:29:53.864338+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":291,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.934240+0000","last_change":"2024-09-21T10:25:48.763096+0000","last_active":"2024-09-21T10:30:00.934240+0000","last_peered":"2024-09-21T10:30:00.934240+0000","last_clean":"2024-09-21T10:30:00.934240+0000","last_became_active":"2024-09-21T10:25:48.761327+0000","last_became_peered":"2024-09-21T10:25:48.761327+0000","last_unstale":"2024-09-21T10:30:00.934240+0000","last_undegraded":"2024-09-21T10:30:00.934240+0000","last_fullsized":"2024-09-21T10:30:00.934240+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:23.154333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"427'3784","reported_seq":3811,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:01.010132+0000","last_change":"2024-09-21T10:29:18.323639+0000","last_active":"2024-09-21T10:30:01.010132+0000","last_peered":"2024-09-21T10:30:01.010132+0000","last_clean":"2024-09-21T10:30:01.010132+0000","last_became_active":"2024-09-21T10:29:18.323318+0000","last_became_peered":"2024-09-21T10:29:18.323318+0000","last_unstale":"2024-09-21T10:30:01.010132+0000","last_undegraded":"2024-09-21T10:30:01.010132+0000","last_fullsized":"2024-09-21T10:30:01.010132+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3784,"log_dups_size":0,"ondisk_log_size":3784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:27:31.319819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247988224,"num_objects":3784,"num_object_clones":0,"num_object_copies":11352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3784,"num_write_kb":242176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":713,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.863174+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:29:53.863174+0000","last_peered":"2024-09-21T10:29:53.863174+0000","last_clean":"2024-09-21T10:29:53.863174+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:29:53.863174+0000","last_undegraded":"2024-09-21T10:29:53.863174+0000","last_fullsized":"2024-09-21T10:29:53.863174+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":793,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861597+0000","last_change":"2024-09-21T10:25:48.755999+0000","last_active":"2024-09-21T10:29:53.861597+0000","last_peered":"2024-09-21T10:29:53.861597+0000","last_clean":"2024-09-21T10:29:53.861597+0000","last_became_active":"2024-09-21T10:25:48.754257+0000","last_became_peered":"2024-09-21T10:25:48.754257+0000","last_unstale":"2024-09-21T10:29:53.861597+0000","last_undegraded":"2024-09-21T10:29:53.861597+0000","last_fullsized":"2024-09-21T10:29:53.861597+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:06.025116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":125,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861773+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:29:53.861773+0000","last_peered":"2024-09-21T10:29:53.861773+0000","last_clean":"2024-09-21T10:29:53.861773+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:29:53.861773+0000","last_undegraded":"2024-09-21T10:29:53.861773+0000","last_fullsized":"2024-09-21T10:29:53.861773+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"427'4143","reported_seq":4168,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.684684+0000","last_change":"2024-09-21T10:29:18.322964+0000","last_active":"2024-09-21T10:30:04.684684+0000","last_peered":"2024-09-21T10:30:04.684684+0000","last_clean":"2024-09-21T10:30:04.684684+0000","last_became_active":"2024-09-21T10:29:18.322677+0000","last_became_peered":"2024-09-21T10:29:18.322677+0000","last_unstale":"2024-09-21T10:30:04.684684+0000","last_undegraded":"2024-09-21T10:30:04.684684+0000","last_fullsized":"2024-09-21T10:30:04.684684+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":4143,"log_dups_size":0,"ondisk_log_size":4143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:06.684046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271515648,"num_objects":4143,"num_object_clones":0,"num_object_copies":12429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4143,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":486,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.863838+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:29:53.863838+0000","last_peered":"2024-09-21T10:29:53.863838+0000","last_clean":"2024-09-21T10:29:53.863838+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:29:53.863838+0000","last_undegraded":"2024-09-21T10:29:53.863838+0000","last_fullsized":"2024-09-21T10:29:53.863838+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":438,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.872152+0000","last_change":"2024-09-21T10:19:50.589605+0000","last_active":"2024-09-21T10:29:53.872152+0000","last_peered":"2024-09-21T10:29:53.872152+0000","last_clean":"2024-09-21T10:29:53.872152+0000","last_became_active":"2024-09-21T10:19:50.589206+0000","last_became_peered":"2024-09-21T10:19:50.589206+0000","last_unstale":"2024-09-21T10:29:53.872152+0000","last_undegraded":"2024-09-21T10:29:53.872152+0000","last_fullsized":"2024-09-21T10:29:53.872152+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:14.024238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.860757+0000","last_change":"2024-09-21T10:19:50.590190+0000","last_active":"2024-09-21T10:29:53.860757+0000","last_peered":"2024-09-21T10:29:53.860757+0000","last_clean":"2024-09-21T10:29:53.860757+0000","last_became_active":"2024-09-21T10:19:50.589825+0000","last_became_peered":"2024-09-21T10:19:50.589825+0000","last_unstale":"2024-09-21T10:29:53.860757+0000","last_undegraded":"2024-09-21T10:29:53.860757+0000","last_fullsized":"2024-09-21T10:29:53.860757+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:19.544473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"427'4030","reported_seq":4047,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.158449+0000","last_change":"2024-09-21T10:29:18.322112+0000","last_active":"2024-09-21T10:30:04.158449+0000","last_peered":"2024-09-21T10:30:04.158449+0000","last_clean":"2024-09-21T10:30:04.158449+0000","last_became_active":"2024-09-21T10:29:18.321784+0000","last_became_peered":"2024-09-21T10:29:18.321784+0000","last_unstale":"2024-09-21T10:30:04.158449+0000","last_undegraded":"2024-09-21T10:30:04.158449+0000","last_fullsized":"2024-09-21T10:30:04.158449+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":4030,"log_dups_size":0,"ondisk_log_size":4030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:10.300972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264110080,"num_objects":4030,"num_object_clones":0,"num_object_copies":12090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4030,"num_write_kb":257920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":125,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.865664+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:29:53.865664+0000","last_peered":"2024-09-21T10:29:53.865664+0000","last_clean":"2024-09-21T10:29:53.865664+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:29:53.865664+0000","last_undegraded":"2024-09-21T10:29:53.865664+0000","last_fullsized":"2024-09-21T10:29:53.865664+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":125,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.862432+0000","last_change":"2024-09-21T10:19:50.590271+0000","last_active":"2024-09-21T10:29:53.862432+0000","last_peered":"2024-09-21T10:29:53.862432+0000","last_clean":"2024-09-21T10:29:53.862432+0000","last_became_active":"2024-09-21T10:19:50.589932+0000","last_became_peered":"2024-09-21T10:19:50.589932+0000","last_unstale":"2024-09-21T10:29:53.862432+0000","last_undegraded":"2024-09-21T10:29:53.862432+0000","last_fullsized":"2024-09-21T10:29:53.862432+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:45.282221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"427'3991","reported_seq":4008,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.064351+0000","last_change":"2024-09-21T10:29:18.319916+0000","last_active":"2024-09-21T10:30:04.064351+0000","last_peered":"2024-09-21T10:30:04.064351+0000","last_clean":"2024-09-21T10:30:04.064351+0000","last_became_active":"2024-09-21T10:29:18.319395+0000","last_became_peered":"2024-09-21T10:29:18.319395+0000","last_unstale":"2024-09-21T10:30:04.064351+0000","last_undegraded":"2024-09-21T10:30:04.064351+0000","last_fullsized":"2024-09-21T10:30:04.064351+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3991,"log_dups_size":0,"ondisk_log_size":3991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:46.166177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261554176,"num_objects":3991,"num_object_clones":0,"num_object_copies":11973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3991,"num_write_kb":255424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":664,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.864394+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:29:53.864394+0000","last_peered":"2024-09-21T10:29:53.864394+0000","last_clean":"2024-09-21T10:29:53.864394+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:29:53.864394+0000","last_undegraded":"2024-09-21T10:29:53.864394+0000","last_fullsized":"2024-09-21T10:29:53.864394+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":761,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861700+0000","last_change":"2024-09-21T10:21:34.871243+0000","last_active":"2024-09-21T10:29:53.861700+0000","last_peered":"2024-09-21T10:29:53.861700+0000","last_clean":"2024-09-21T10:29:53.861700+0000","last_became_active":"2024-09-21T10:21:34.869472+0000","last_became_peered":"2024-09-21T10:21:34.869472+0000","last_unstale":"2024-09-21T10:29:53.861700+0000","last_undegraded":"2024-09-21T10:29:53.861700+0000","last_fullsized":"2024-09-21T10:29:53.861700+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:38.269886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":455,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.932287+0000","last_change":"2024-09-21T10:19:50.593176+0000","last_active":"2024-09-21T10:30:00.932287+0000","last_peered":"2024-09-21T10:30:00.932287+0000","last_clean":"2024-09-21T10:30:00.932287+0000","last_became_active":"2024-09-21T10:19:50.592773+0000","last_became_peered":"2024-09-21T10:19:50.592773+0000","last_unstale":"2024-09-21T10:30:00.932287+0000","last_undegraded":"2024-09-21T10:30:00.932287+0000","last_fullsized":"2024-09-21T10:30:00.932287+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:54.546869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"427'4052","reported_seq":4069,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.164795+0000","last_change":"2024-09-21T10:29:18.321006+0000","last_active":"2024-09-21T10:30:04.164795+0000","last_peered":"2024-09-21T10:30:04.164795+0000","last_clean":"2024-09-21T10:30:04.164795+0000","last_became_active":"2024-09-21T10:29:18.320643+0000","last_became_peered":"2024-09-21T10:29:18.320643+0000","last_unstale":"2024-09-21T10:30:04.164795+0000","last_undegraded":"2024-09-21T10:30:04.164795+0000","last_fullsized":"2024-09-21T10:30:04.164795+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":4052,"log_dups_size":0,"ondisk_log_size":4052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:44.782234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265551872,"num_objects":4052,"num_object_clones":0,"num_object_copies":12156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4052,"num_write_kb":259328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":713,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861353+0000","last_change":"2024-09-21T10:25:48.754725+0000","last_active":"2024-09-21T10:29:53.861353+0000","last_peered":"2024-09-21T10:29:53.861353+0000","last_clean":"2024-09-21T10:29:53.861353+0000","last_became_active":"2024-09-21T10:25:48.754341+0000","last_became_peered":"2024-09-21T10:25:48.754341+0000","last_unstale":"2024-09-21T10:29:53.861353+0000","last_undegraded":"2024-09-21T10:29:53.861353+0000","last_fullsized":"2024-09-21T10:29:53.861353+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:25.191555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:25.191555+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:08.981054+0000","scrub_duration":0.0015044590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":741,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.864658+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:29:53.864658+0000","last_peered":"2024-09-21T10:29:53.864658+0000","last_clean":"2024-09-21T10:29:53.864658+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:29:53.864658+0000","last_undegraded":"2024-09-21T10:29:53.864658+0000","last_fullsized":"2024-09-21T10:29:53.864658+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":692,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861031+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:29:53.861031+0000","last_peered":"2024-09-21T10:29:53.861031+0000","last_clean":"2024-09-21T10:29:53.861031+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:29:53.861031+0000","last_undegraded":"2024-09-21T10:29:53.861031+0000","last_fullsized":"2024-09-21T10:29:53.861031+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":404,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.869939+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:29:53.869939+0000","last_peered":"2024-09-21T10:29:53.869939+0000","last_clean":"2024-09-21T10:29:53.869939+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:29:53.869939+0000","last_undegraded":"2024-09-21T10:29:53.869939+0000","last_fullsized":"2024-09-21T10:29:53.869939+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"427'3924","reported_seq":3950,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:02.220422+0000","last_change":"2024-09-21T10:29:18.319178+0000","last_active":"2024-09-21T10:30:02.220422+0000","last_peered":"2024-09-21T10:30:02.220422+0000","last_clean":"2024-09-21T10:30:02.220422+0000","last_became_active":"2024-09-21T10:29:18.318926+0000","last_became_peered":"2024-09-21T10:29:18.318926+0000","last_unstale":"2024-09-21T10:30:02.220422+0000","last_undegraded":"2024-09-21T10:30:02.220422+0000","last_fullsized":"2024-09-21T10:30:02.220422+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3924,"log_dups_size":0,"ondisk_log_size":3924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:22.626306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257163264,"num_objects":3924,"num_object_clones":0,"num_object_copies":11772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3924,"num_write_kb":251136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":711,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.932146+0000","last_change":"2024-09-21T10:24:27.359111+0000","last_active":"2024-09-21T10:30:00.932146+0000","last_peered":"2024-09-21T10:30:00.932146+0000","last_clean":"2024-09-21T10:30:00.932146+0000","last_became_active":"2024-09-21T10:24:24.825850+0000","last_became_peered":"2024-09-21T10:24:24.825850+0000","last_unstale":"2024-09-21T10:30:00.932146+0000","last_undegraded":"2024-09-21T10:30:00.932146+0000","last_fullsized":"2024-09-21T10:30:00.932146+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:27.359065+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:24:27.359065+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:06.871319+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1085,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.862120+0000","last_change":"2024-09-21T10:25:48.755667+0000","last_active":"2024-09-21T10:29:53.862120+0000","last_peered":"2024-09-21T10:29:53.862120+0000","last_clean":"2024-09-21T10:29:53.862120+0000","last_became_active":"2024-09-21T10:25:48.755060+0000","last_became_peered":"2024-09-21T10:25:48.755060+0000","last_unstale":"2024-09-21T10:29:53.862120+0000","last_undegraded":"2024-09-21T10:29:53.862120+0000","last_fullsized":"2024-09-21T10:29:53.862120+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:09.230030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":294,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.933700+0000","last_change":"2024-09-21T10:25:48.754659+0000","last_active":"2024-09-21T10:30:00.933700+0000","last_peered":"2024-09-21T10:30:00.933700+0000","last_clean":"2024-09-21T10:30:00.933700+0000","last_became_active":"2024-09-21T10:25:48.754230+0000","last_became_peered":"2024-09-21T10:25:48.754230+0000","last_unstale":"2024-09-21T10:30:00.933700+0000","last_undegraded":"2024-09-21T10:30:00.933700+0000","last_fullsized":"2024-09-21T10:30:00.933700+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:12.555868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":100,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.933788+0000","last_change":"2024-09-21T10:25:48.761249+0000","last_active":"2024-09-21T10:30:00.933788+0000","last_peered":"2024-09-21T10:30:00.933788+0000","last_clean":"2024-09-21T10:30:00.933788+0000","last_became_active":"2024-09-21T10:25:48.759359+0000","last_became_peered":"2024-09-21T10:25:48.759359+0000","last_unstale":"2024-09-21T10:30:00.933788+0000","last_undegraded":"2024-09-21T10:30:00.933788+0000","last_fullsized":"2024-09-21T10:30:00.933788+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:33.510936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"427'3862","reported_seq":3879,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:01.059182+0000","last_change":"2024-09-21T10:29:18.320919+0000","last_active":"2024-09-21T10:30:01.059182+0000","last_peered":"2024-09-21T10:30:01.059182+0000","last_clean":"2024-09-21T10:30:01.059182+0000","last_became_active":"2024-09-21T10:29:18.320597+0000","last_became_peered":"2024-09-21T10:29:18.320597+0000","last_unstale":"2024-09-21T10:30:01.059182+0000","last_undegraded":"2024-09-21T10:30:01.059182+0000","last_fullsized":"2024-09-21T10:30:01.059182+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3862,"log_dups_size":0,"ondisk_log_size":3862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:30.147686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253100032,"num_objects":3862,"num_object_clones":0,"num_object_copies":11586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3862,"num_write_kb":247168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":729,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:59.295718+0000","last_change":"2024-09-21T10:25:48.755509+0000","last_active":"2024-09-21T10:29:59.295718+0000","last_peered":"2024-09-21T10:29:59.295718+0000","last_clean":"2024-09-21T10:29:59.295718+0000","last_became_active":"2024-09-21T10:25:48.754558+0000","last_became_peered":"2024-09-21T10:25:48.754558+0000","last_unstale":"2024-09-21T10:29:59.295718+0000","last_undegraded":"2024-09-21T10:29:59.295718+0000","last_fullsized":"2024-09-21T10:29:59.295718+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:15.203886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:15.203886+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:11.262503+0000","scrub_duration":0.0013422639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1294,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.912790+0000","last_change":"2024-09-21T10:25:48.753359+0000","last_active":"2024-09-21T10:30:00.912790+0000","last_peered":"2024-09-21T10:30:00.912790+0000","last_clean":"2024-09-21T10:30:00.912790+0000","last_became_active":"2024-09-21T10:25:48.752653+0000","last_became_peered":"2024-09-21T10:25:48.752653+0000","last_unstale":"2024-09-21T10:30:00.912790+0000","last_undegraded":"2024-09-21T10:30:00.912790+0000","last_fullsized":"2024-09-21T10:30:00.912790+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:55.095131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":442,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.932040+0000","last_change":"2024-09-21T10:13:09.581278+0000","last_active":"2024-09-21T10:30:00.932040+0000","last_peered":"2024-09-21T10:30:00.932040+0000","last_clean":"2024-09-21T10:30:00.932040+0000","last_became_active":"2024-09-21T10:13:09.580960+0000","last_became_peered":"2024-09-21T10:13:09.580960+0000","last_unstale":"2024-09-21T10:30:00.932040+0000","last_undegraded":"2024-09-21T10:30:00.932040+0000","last_fullsized":"2024-09-21T10:30:00.932040+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:21.061808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":482,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.932398+0000","last_change":"2024-09-21T10:25:48.766755+0000","last_active":"2024-09-21T10:30:00.932398+0000","last_peered":"2024-09-21T10:30:00.932398+0000","last_clean":"2024-09-21T10:30:00.932398+0000","last_became_active":"2024-09-21T10:25:48.766298+0000","last_became_peered":"2024-09-21T10:25:48.766298+0000","last_unstale":"2024-09-21T10:30:00.932398+0000","last_undegraded":"2024-09-21T10:30:00.932398+0000","last_fullsized":"2024-09-21T10:30:00.932398+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:18.788447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"427'4030","reported_seq":4047,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.146876+0000","last_change":"2024-09-21T10:29:18.323747+0000","last_active":"2024-09-21T10:30:04.146876+0000","last_peered":"2024-09-21T10:30:04.146876+0000","last_clean":"2024-09-21T10:30:04.146876+0000","last_became_active":"2024-09-21T10:29:18.323433+0000","last_became_peered":"2024-09-21T10:29:18.323433+0000","last_unstale":"2024-09-21T10:30:04.146876+0000","last_undegraded":"2024-09-21T10:30:04.146876+0000","last_fullsized":"2024-09-21T10:30:04.146876+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":4030,"log_dups_size":0,"ondisk_log_size":4030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:50.727742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264110080,"num_objects":4030,"num_object_clones":0,"num_object_copies":12090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4030,"num_write_kb":257920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":679,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.866535+0000","last_change":"2024-09-21T10:25:48.757672+0000","last_active":"2024-09-21T10:29:53.866535+0000","last_peered":"2024-09-21T10:29:53.866535+0000","last_clean":"2024-09-21T10:29:53.866535+0000","last_became_active":"2024-09-21T10:25:48.757296+0000","last_became_peered":"2024-09-21T10:25:48.757296+0000","last_unstale":"2024-09-21T10:29:53.866535+0000","last_undegraded":"2024-09-21T10:29:53.866535+0000","last_fullsized":"2024-09-21T10:29:53.866535+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:31.244809+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:24:31.244809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:32.654430+0000","scrub_duration":0.0015828649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1172,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.864809+0000","last_change":"2024-09-21T10:20:13.085689+0000","last_active":"2024-09-21T10:29:53.864809+0000","last_peered":"2024-09-21T10:29:53.864809+0000","last_clean":"2024-09-21T10:29:53.864809+0000","last_became_active":"2024-09-21T10:20:13.085184+0000","last_became_peered":"2024-09-21T10:20:13.085184+0000","last_unstale":"2024-09-21T10:29:53.864809+0000","last_undegraded":"2024-09-21T10:29:53.864809+0000","last_fullsized":"2024-09-21T10:29:53.864809+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:32.785493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":140,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.860185+0000","last_change":"2024-09-21T10:25:48.753400+0000","last_active":"2024-09-21T10:29:53.860185+0000","last_peered":"2024-09-21T10:29:53.860185+0000","last_clean":"2024-09-21T10:29:53.860185+0000","last_became_active":"2024-09-21T10:25:48.753088+0000","last_became_peered":"2024-09-21T10:25:48.753088+0000","last_unstale":"2024-09-21T10:29:53.860185+0000","last_undegraded":"2024-09-21T10:29:53.860185+0000","last_fullsized":"2024-09-21T10:29:53.860185+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:49.107566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":442,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.932243+0000","last_change":"2024-09-21T10:13:09.585131+0000","last_active":"2024-09-21T10:30:00.932243+0000","last_peered":"2024-09-21T10:30:00.932243+0000","last_clean":"2024-09-21T10:30:00.932243+0000","last_became_active":"2024-09-21T10:13:09.584795+0000","last_became_peered":"2024-09-21T10:13:09.584795+0000","last_unstale":"2024-09-21T10:30:00.932243+0000","last_undegraded":"2024-09-21T10:30:00.932243+0000","last_fullsized":"2024-09-21T10:30:00.932243+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:13.001923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"427'3896","reported_seq":3913,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.271885+0000","last_change":"2024-09-21T10:29:18.323540+0000","last_active":"2024-09-21T10:30:04.271885+0000","last_peered":"2024-09-21T10:30:04.271885+0000","last_clean":"2024-09-21T10:30:04.271885+0000","last_became_active":"2024-09-21T10:29:18.323125+0000","last_became_peered":"2024-09-21T10:29:18.323125+0000","last_unstale":"2024-09-21T10:30:04.271885+0000","last_undegraded":"2024-09-21T10:30:04.271885+0000","last_fullsized":"2024-09-21T10:30:04.271885+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3896,"log_dups_size":0,"ondisk_log_size":3896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:33.821481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255328256,"num_objects":3896,"num_object_clones":0,"num_object_copies":11688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3896,"num_write_kb":249344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":512,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.870099+0000","last_change":"2024-09-21T10:24:24.828950+0000","last_active":"2024-09-21T10:29:53.870099+0000","last_peered":"2024-09-21T10:29:53.870099+0000","last_clean":"2024-09-21T10:29:53.870099+0000","last_became_active":"2024-09-21T10:24:24.828550+0000","last_became_peered":"2024-09-21T10:24:24.828550+0000","last_unstale":"2024-09-21T10:29:53.870099+0000","last_undegraded":"2024-09-21T10:29:53.870099+0000","last_fullsized":"2024-09-21T10:29:53.870099+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:19:04.424309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:19:04.424309+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:29.313198+0000","scrub_duration":0.001370488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":377,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.864194+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:29:53.864194+0000","last_peered":"2024-09-21T10:29:53.864194+0000","last_clean":"2024-09-21T10:29:53.864194+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:29:53.864194+0000","last_undegraded":"2024-09-21T10:29:53.864194+0000","last_fullsized":"2024-09-21T10:29:53.864194+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":1062,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.932401+0000","last_change":"2024-09-21T10:25:48.760586+0000","last_active":"2024-09-21T10:30:00.932401+0000","last_peered":"2024-09-21T10:30:00.932401+0000","last_clean":"2024-09-21T10:30:00.932401+0000","last_became_active":"2024-09-21T10:25:48.760042+0000","last_became_peered":"2024-09-21T10:25:48.760042+0000","last_unstale":"2024-09-21T10:30:00.932401+0000","last_undegraded":"2024-09-21T10:30:00.932401+0000","last_fullsized":"2024-09-21T10:30:00.932401+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:41.526062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":478,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.932289+0000","last_change":"2024-09-21T10:20:14.261720+0000","last_active":"2024-09-21T10:30:00.932289+0000","last_peered":"2024-09-21T10:30:00.932289+0000","last_clean":"2024-09-21T10:30:00.932289+0000","last_became_active":"2024-09-21T10:20:14.261405+0000","last_became_peered":"2024-09-21T10:20:14.261405+0000","last_unstale":"2024-09-21T10:30:00.932289+0000","last_undegraded":"2024-09-21T10:30:00.932289+0000","last_fullsized":"2024-09-21T10:30:00.932289+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:34.402258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"427'4022","reported_seq":4038,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.165739+0000","last_change":"2024-09-21T10:29:18.320870+0000","last_active":"2024-09-21T10:30:04.165739+0000","last_peered":"2024-09-21T10:30:04.165739+0000","last_clean":"2024-09-21T10:30:04.165739+0000","last_became_active":"2024-09-21T10:29:18.320565+0000","last_became_peered":"2024-09-21T10:29:18.320565+0000","last_unstale":"2024-09-21T10:30:04.165739+0000","last_undegraded":"2024-09-21T10:30:04.165739+0000","last_fullsized":"2024-09-21T10:30:04.165739+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":4022,"log_dups_size":0,"ondisk_log_size":4022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:51.308850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263585792,"num_objects":4022,"num_object_clones":0,"num_object_copies":12066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4022,"num_write_kb":257408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":527,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.869557+0000","last_change":"2024-09-21T10:25:48.752455+0000","last_active":"2024-09-21T10:29:53.869557+0000","last_peered":"2024-09-21T10:29:53.869557+0000","last_clean":"2024-09-21T10:29:53.869557+0000","last_became_active":"2024-09-21T10:25:48.752056+0000","last_became_peered":"2024-09-21T10:25:48.752056+0000","last_unstale":"2024-09-21T10:29:53.869557+0000","last_undegraded":"2024-09-21T10:29:53.869557+0000","last_fullsized":"2024-09-21T10:29:53.869557+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:13.176002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:13.176002+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:15.886186+0000","scrub_duration":0.0018106349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":460,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.932397+0000","last_change":"2024-09-21T10:20:13.104426+0000","last_active":"2024-09-21T10:30:00.932397+0000","last_peered":"2024-09-21T10:30:00.932397+0000","last_clean":"2024-09-21T10:30:00.932397+0000","last_became_active":"2024-09-21T10:20:13.104076+0000","last_became_peered":"2024-09-21T10:20:13.104076+0000","last_unstale":"2024-09-21T10:30:00.932397+0000","last_undegraded":"2024-09-21T10:30:00.932397+0000","last_fullsized":"2024-09-21T10:30:00.932397+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.522650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":1011,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861735+0000","last_change":"2024-09-21T10:25:48.755967+0000","last_active":"2024-09-21T10:29:53.861735+0000","last_peered":"2024-09-21T10:29:53.861735+0000","last_clean":"2024-09-21T10:29:53.861735+0000","last_became_active":"2024-09-21T10:25:48.755037+0000","last_became_peered":"2024-09-21T10:25:48.755037+0000","last_unstale":"2024-09-21T10:29:53.861735+0000","last_undegraded":"2024-09-21T10:29:53.861735+0000","last_fullsized":"2024-09-21T10:29:53.861735+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:26.414898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":309,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.933947+0000","last_change":"2024-09-21T10:25:48.767902+0000","last_active":"2024-09-21T10:30:00.933947+0000","last_peered":"2024-09-21T10:30:00.933947+0000","last_clean":"2024-09-21T10:30:00.933947+0000","last_became_active":"2024-09-21T10:25:48.765443+0000","last_became_peered":"2024-09-21T10:25:48.765443+0000","last_unstale":"2024-09-21T10:30:00.933947+0000","last_undegraded":"2024-09-21T10:30:00.933947+0000","last_fullsized":"2024-09-21T10:30:00.933947+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:26.622825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"427'4017","reported_seq":4034,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.039001+0000","last_change":"2024-09-21T10:29:18.321928+0000","last_active":"2024-09-21T10:30:04.039001+0000","last_peered":"2024-09-21T10:30:04.039001+0000","last_clean":"2024-09-21T10:30:04.039001+0000","last_became_active":"2024-09-21T10:29:18.321303+0000","last_became_peered":"2024-09-21T10:29:18.321303+0000","last_unstale":"2024-09-21T10:30:04.039001+0000","last_undegraded":"2024-09-21T10:30:04.039001+0000","last_fullsized":"2024-09-21T10:30:04.039001+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":4017,"log_dups_size":0,"ondisk_log_size":4017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:25.083294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263258112,"num_objects":4017,"num_object_clones":0,"num_object_copies":12051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4017,"num_write_kb":257088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":664,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.860951+0000","last_change":"2024-09-21T10:24:26.255840+0000","last_active":"2024-09-21T10:29:53.860951+0000","last_peered":"2024-09-21T10:29:53.860951+0000","last_clean":"2024-09-21T10:29:53.860951+0000","last_became_active":"2024-09-21T10:24:24.825549+0000","last_became_peered":"2024-09-21T10:24:24.825549+0000","last_unstale":"2024-09-21T10:29:53.860951+0000","last_undegraded":"2024-09-21T10:29:53.860951+0000","last_fullsized":"2024-09-21T10:29:53.860951+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:26.255753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.255753+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:25.638856+0000","scrub_duration":0.0016223839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":777,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.862448+0000","last_change":"2024-09-21T10:25:48.758590+0000","last_active":"2024-09-21T10:29:53.862448+0000","last_peered":"2024-09-21T10:29:53.862448+0000","last_clean":"2024-09-21T10:29:53.862448+0000","last_became_active":"2024-09-21T10:25:48.758189+0000","last_became_peered":"2024-09-21T10:25:48.758189+0000","last_unstale":"2024-09-21T10:29:53.862448+0000","last_undegraded":"2024-09-21T10:29:53.862448+0000","last_fullsized":"2024-09-21T10:29:53.862448+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:24.126237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":1113,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.932596+0000","last_change":"2024-09-21T10:19:53.569601+0000","last_active":"2024-09-21T10:30:00.932596+0000","last_peered":"2024-09-21T10:30:00.932596+0000","last_clean":"2024-09-21T10:30:00.932596+0000","last_became_active":"2024-09-21T10:19:53.568838+0000","last_became_peered":"2024-09-21T10:19:53.568838+0000","last_unstale":"2024-09-21T10:30:00.932596+0000","last_undegraded":"2024-09-21T10:30:00.932596+0000","last_fullsized":"2024-09-21T10:30:00.932596+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:18.989240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":773,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861544+0000","last_change":"2024-09-21T10:25:48.754966+0000","last_active":"2024-09-21T10:29:53.861544+0000","last_peered":"2024-09-21T10:29:53.861544+0000","last_clean":"2024-09-21T10:29:53.861544+0000","last_became_active":"2024-09-21T10:25:48.754152+0000","last_became_peered":"2024-09-21T10:25:48.754152+0000","last_unstale":"2024-09-21T10:29:53.861544+0000","last_undegraded":"2024-09-21T10:29:53.861544+0000","last_fullsized":"2024-09-21T10:29:53.861544+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:33:28.795713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"427'3945","reported_seq":3962,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:04.151471+0000","last_change":"2024-09-21T10:29:18.318036+0000","last_active":"2024-09-21T10:30:04.151471+0000","last_peered":"2024-09-21T10:30:04.151471+0000","last_clean":"2024-09-21T10:30:04.151471+0000","last_became_active":"2024-09-21T10:29:18.317716+0000","last_became_peered":"2024-09-21T10:29:18.317716+0000","last_unstale":"2024-09-21T10:30:04.151471+0000","last_undegraded":"2024-09-21T10:30:04.151471+0000","last_fullsized":"2024-09-21T10:30:04.151471+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":3945,"log_dups_size":0,"ondisk_log_size":3945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:58.738776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258539520,"num_objects":3945,"num_object_clones":0,"num_object_copies":11835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3945,"num_write_kb":252480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":745,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.863558+0000","last_change":"2024-09-21T10:24:26.926717+0000","last_active":"2024-09-21T10:29:53.863558+0000","last_peered":"2024-09-21T10:29:53.863558+0000","last_clean":"2024-09-21T10:29:53.863558+0000","last_became_active":"2024-09-21T10:24:24.828205+0000","last_became_peered":"2024-09-21T10:24:24.828205+0000","last_unstale":"2024-09-21T10:29:53.863558+0000","last_undegraded":"2024-09-21T10:29:53.863558+0000","last_fullsized":"2024-09-21T10:29:53.863558+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:24:26.926661+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:24:26.926661+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:47.946104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":129,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.934351+0000","last_change":"2024-09-21T10:25:48.762565+0000","last_active":"2024-09-21T10:30:00.934351+0000","last_peered":"2024-09-21T10:30:00.934351+0000","last_clean":"2024-09-21T10:30:00.934351+0000","last_became_active":"2024-09-21T10:25:48.761324+0000","last_became_peered":"2024-09-21T10:25:48.761324+0000","last_unstale":"2024-09-21T10:30:00.934351+0000","last_undegraded":"2024-09-21T10:30:00.934351+0000","last_fullsized":"2024-09-21T10:30:00.934351+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:58.885790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":739,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.870635+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:29:53.870635+0000","last_peered":"2024-09-21T10:29:53.870635+0000","last_clean":"2024-09-21T10:29:53.870635+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:29:53.870635+0000","last_undegraded":"2024-09-21T10:29:53.870635+0000","last_fullsized":"2024-09-21T10:29:53.870635+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":401,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.865856+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:29:53.865856+0000","last_peered":"2024-09-21T10:29:53.865856+0000","last_clean":"2024-09-21T10:29:53.865856+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:29:53.865856+0000","last_undegraded":"2024-09-21T10:29:53.865856+0000","last_fullsized":"2024-09-21T10:29:53.865856+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":425,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.860250+0000","last_change":"2024-09-21T10:19:51.554098+0000","last_active":"2024-09-21T10:29:53.860250+0000","last_peered":"2024-09-21T10:29:53.860250+0000","last_clean":"2024-09-21T10:29:53.860250+0000","last_became_active":"2024-09-21T10:19:51.553799+0000","last_became_peered":"2024-09-21T10:19:51.553799+0000","last_unstale":"2024-09-21T10:29:53.860250+0000","last_undegraded":"2024-09-21T10:29:53.860250+0000","last_fullsized":"2024-09-21T10:29:53.860250+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:13.698516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":135,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861856+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:29:53.861856+0000","last_peered":"2024-09-21T10:29:53.861856+0000","last_clean":"2024-09-21T10:29:53.861856+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:29:53.861856+0000","last_undegraded":"2024-09-21T10:29:53.861856+0000","last_fullsized":"2024-09-21T10:29:53.861856+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":129,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.864650+0000","last_change":"2024-09-21T10:20:13.087786+0000","last_active":"2024-09-21T10:29:53.864650+0000","last_peered":"2024-09-21T10:29:53.864650+0000","last_clean":"2024-09-21T10:29:53.864650+0000","last_became_active":"2024-09-21T10:20:13.087317+0000","last_became_peered":"2024-09-21T10:20:13.087317+0000","last_unstale":"2024-09-21T10:29:53.864650+0000","last_undegraded":"2024-09-21T10:29:53.864650+0000","last_fullsized":"2024-09-21T10:29:53.864650+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:55.709613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":108,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861343+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:29:53.861343+0000","last_peered":"2024-09-21T10:29:53.861343+0000","last_clean":"2024-09-21T10:29:53.861343+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:29:53.861343+0000","last_undegraded":"2024-09-21T10:29:53.861343+0000","last_fullsized":"2024-09-21T10:29:53.861343+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":137,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.863912+0000","last_change":"2024-09-21T10:21:34.866527+0000","last_active":"2024-09-21T10:29:53.863912+0000","last_peered":"2024-09-21T10:29:53.863912+0000","last_clean":"2024-09-21T10:29:53.863912+0000","last_became_active":"2024-09-21T10:21:34.866197+0000","last_became_peered":"2024-09-21T10:21:34.866197+0000","last_unstale":"2024-09-21T10:29:53.863912+0000","last_undegraded":"2024-09-21T10:29:53.863912+0000","last_fullsized":"2024-09-21T10:29:53.863912+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:10.152695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":685,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861123+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:29:53.861123+0000","last_peered":"2024-09-21T10:29:53.861123+0000","last_clean":"2024-09-21T10:29:53.861123+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:29:53.861123+0000","last_undegraded":"2024-09-21T10:29:53.861123+0000","last_fullsized":"2024-09-21T10:29:53.861123+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":306,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.934386+0000","last_change":"2024-09-21T10:25:48.765786+0000","last_active":"2024-09-21T10:30:00.934386+0000","last_peered":"2024-09-21T10:30:00.934386+0000","last_clean":"2024-09-21T10:30:00.934386+0000","last_became_active":"2024-09-21T10:25:48.764011+0000","last_became_peered":"2024-09-21T10:25:48.764011+0000","last_unstale":"2024-09-21T10:30:00.934386+0000","last_undegraded":"2024-09-21T10:30:00.934386+0000","last_fullsized":"2024-09-21T10:30:00.934386+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:13.352641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":745,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.860834+0000","last_change":"2024-09-21T10:13:09.579189+0000","last_active":"2024-09-21T10:29:53.860834+0000","last_peered":"2024-09-21T10:29:53.860834+0000","last_clean":"2024-09-21T10:29:53.860834+0000","last_became_active":"2024-09-21T10:13:09.578756+0000","last_became_peered":"2024-09-21T10:13:09.578756+0000","last_unstale":"2024-09-21T10:29:53.860834+0000","last_undegraded":"2024-09-21T10:29:53.860834+0000","last_fullsized":"2024-09-21T10:29:53.860834+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:00.054820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":444,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:54.110312+0000","last_change":"2024-09-21T10:20:13.089062+0000","last_active":"2024-09-21T10:29:54.110312+0000","last_peered":"2024-09-21T10:29:54.110312+0000","last_clean":"2024-09-21T10:29:54.110312+0000","last_became_active":"2024-09-21T10:20:13.088766+0000","last_became_peered":"2024-09-21T10:20:13.088766+0000","last_unstale":"2024-09-21T10:29:54.110312+0000","last_undegraded":"2024-09-21T10:29:54.110312+0000","last_fullsized":"2024-09-21T10:29:54.110312+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:25.868592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":772,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861586+0000","last_change":"2024-09-21T10:25:48.755341+0000","last_active":"2024-09-21T10:29:53.861586+0000","last_peered":"2024-09-21T10:29:53.861586+0000","last_clean":"2024-09-21T10:29:53.861586+0000","last_became_active":"2024-09-21T10:25:48.754620+0000","last_became_peered":"2024-09-21T10:25:48.754620+0000","last_unstale":"2024-09-21T10:29:53.861586+0000","last_undegraded":"2024-09-21T10:29:53.861586+0000","last_fullsized":"2024-09-21T10:29:53.861586+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:57:21.296549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":340,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.933882+0000","last_change":"2024-09-21T10:25:48.765683+0000","last_active":"2024-09-21T10:30:00.933882+0000","last_peered":"2024-09-21T10:30:00.933882+0000","last_clean":"2024-09-21T10:30:00.933882+0000","last_became_active":"2024-09-21T10:25:48.765281+0000","last_became_peered":"2024-09-21T10:25:48.765281+0000","last_unstale":"2024-09-21T10:30:00.933882+0000","last_undegraded":"2024-09-21T10:30:00.933882+0000","last_fullsized":"2024-09-21T10:30:00.933882+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:41.798683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":461,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.931923+0000","last_change":"2024-09-21T10:19:50.591365+0000","last_active":"2024-09-21T10:30:00.931923+0000","last_peered":"2024-09-21T10:30:00.931923+0000","last_clean":"2024-09-21T10:30:00.931923+0000","last_became_active":"2024-09-21T10:19:50.590981+0000","last_became_peered":"2024-09-21T10:19:50.590981+0000","last_unstale":"2024-09-21T10:30:00.931923+0000","last_undegraded":"2024-09-21T10:30:00.931923+0000","last_fullsized":"2024-09-21T10:30:00.931923+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:30.916108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":751,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.863083+0000","last_change":"2024-09-21T10:19:50.586841+0000","last_active":"2024-09-21T10:29:53.863083+0000","last_peered":"2024-09-21T10:29:53.863083+0000","last_clean":"2024-09-21T10:29:53.863083+0000","last_became_active":"2024-09-21T10:19:50.586458+0000","last_became_peered":"2024-09-21T10:19:50.586458+0000","last_unstale":"2024-09-21T10:29:53.863083+0000","last_undegraded":"2024-09-21T10:29:53.863083+0000","last_fullsized":"2024-09-21T10:29:53.863083+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:59.897738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":432,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.860014+0000","last_change":"2024-09-21T10:19:51.560158+0000","last_active":"2024-09-21T10:29:53.860014+0000","last_peered":"2024-09-21T10:29:53.860014+0000","last_clean":"2024-09-21T10:29:53.860014+0000","last_became_active":"2024-09-21T10:19:51.558168+0000","last_became_peered":"2024-09-21T10:19:51.558168+0000","last_unstale":"2024-09-21T10:29:53.860014+0000","last_undegraded":"2024-09-21T10:29:53.860014+0000","last_fullsized":"2024-09-21T10:29:53.860014+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:01.739368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":752,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.911735+0000","last_change":"2024-09-21T10:13:09.575732+0000","last_active":"2024-09-21T10:30:00.911735+0000","last_peered":"2024-09-21T10:30:00.911735+0000","last_clean":"2024-09-21T10:30:00.911735+0000","last_became_active":"2024-09-21T10:13:09.575139+0000","last_became_peered":"2024-09-21T10:13:09.575139+0000","last_unstale":"2024-09-21T10:30:00.911735+0000","last_undegraded":"2024-09-21T10:30:00.911735+0000","last_fullsized":"2024-09-21T10:30:00.911735+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:34.625861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":99,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.934063+0000","last_change":"2024-09-21T10:25:48.767385+0000","last_active":"2024-09-21T10:30:00.934063+0000","last_peered":"2024-09-21T10:30:00.934063+0000","last_clean":"2024-09-21T10:30:00.934063+0000","last_became_active":"2024-09-21T10:25:48.764077+0000","last_became_peered":"2024-09-21T10:25:48.764077+0000","last_unstale":"2024-09-21T10:30:00.934063+0000","last_undegraded":"2024-09-21T10:30:00.934063+0000","last_fullsized":"2024-09-21T10:30:00.934063+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:06:51.969987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.859977+0000","last_change":"2024-09-21T10:19:50.584862+0000","last_active":"2024-09-21T10:29:53.859977+0000","last_peered":"2024-09-21T10:29:53.859977+0000","last_clean":"2024-09-21T10:29:53.859977+0000","last_became_active":"2024-09-21T10:19:50.584503+0000","last_became_peered":"2024-09-21T10:19:50.584503+0000","last_unstale":"2024-09-21T10:29:53.859977+0000","last_undegraded":"2024-09-21T10:29:53.859977+0000","last_fullsized":"2024-09-21T10:29:53.859977+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:34.069753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":368,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.864199+0000","last_change":"2024-09-21T10:20:14.245260+0000","last_active":"2024-09-21T10:29:53.864199+0000","last_peered":"2024-09-21T10:29:53.864199+0000","last_clean":"2024-09-21T10:29:53.864199+0000","last_became_active":"2024-09-21T10:20:14.244904+0000","last_became_peered":"2024-09-21T10:20:14.244904+0000","last_unstale":"2024-09-21T10:29:53.864199+0000","last_undegraded":"2024-09-21T10:29:53.864199+0000","last_fullsized":"2024-09-21T10:29:53.864199+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:54.776338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":140,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.859823+0000","last_change":"2024-09-21T10:25:48.753233+0000","last_active":"2024-09-21T10:29:53.859823+0000","last_peered":"2024-09-21T10:29:53.859823+0000","last_clean":"2024-09-21T10:29:53.859823+0000","last_became_active":"2024-09-21T10:25:48.752932+0000","last_became_peered":"2024-09-21T10:25:48.752932+0000","last_unstale":"2024-09-21T10:29:53.859823+0000","last_undegraded":"2024-09-21T10:29:53.859823+0000","last_fullsized":"2024-09-21T10:29:53.859823+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:49.456475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":346,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.864280+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:29:53.864280+0000","last_peered":"2024-09-21T10:29:53.864280+0000","last_clean":"2024-09-21T10:29:53.864280+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:29:53.864280+0000","last_undegraded":"2024-09-21T10:29:53.864280+0000","last_fullsized":"2024-09-21T10:29:53.864280+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":775,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.861410+0000","last_change":"2024-09-21T10:25:48.755262+0000","last_active":"2024-09-21T10:29:53.861410+0000","last_peered":"2024-09-21T10:29:53.861410+0000","last_clean":"2024-09-21T10:29:53.861410+0000","last_became_active":"2024-09-21T10:25:48.754746+0000","last_became_peered":"2024-09-21T10:25:48.754746+0000","last_unstale":"2024-09-21T10:29:53.861410+0000","last_undegraded":"2024-09-21T10:29:53.861410+0000","last_fullsized":"2024-09-21T10:29:53.861410+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:28:17.312418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":727,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.862155+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:29:53.862155+0000","last_peered":"2024-09-21T10:29:53.862155+0000","last_clean":"2024-09-21T10:29:53.862155+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:29:53.862155+0000","last_undegraded":"2024-09-21T10:29:53.862155+0000","last_fullsized":"2024-09-21T10:29:53.862155+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":125,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-21T10:29:53.859713+0000","last_change":"2024-09-21T10:19:50.582707+0000","last_active":"2024-09-21T10:29:53.859713+0000","last_peered":"2024-09-21T10:29:53.859713+0000","last_clean":"2024-09-21T10:29:53.859713+0000","last_became_active":"2024-09-21T10:19:50.582323+0000","last_became_peered":"2024-09-21T10:19:50.582323+0000","last_unstale":"2024-09-21T10:29:53.859713+0000","last_undegraded":"2024-09-21T10:29:53.859713+0000","last_fullsized":"2024-09-21T10:29:53.859713+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:31.095966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":132,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-21T10:30:00.934246+0000","last_change":"2024-09-21T10:25:48.764108+0000","last_active":"2024-09-21T10:30:00.934246+0000","last_peered":"2024-09-21T10:30:00.934246+0000","last_clean":"2024-09-21T10:30:00.934246+0000","last_became_active":"2024-09-21T10:25:48.761361+0000","last_became_peered":"2024-09-21T10:25:48.761361+0000","last_unstale":"2024-09-21T10:30:00.934246+0000","last_undegraded":"2024-09-21T10:30:00.934246+0000","last_fullsized":"2024-09-21T10:30:00.934246+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:49.188529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4143448064,"num_objects":63224,"num_object_clones":0,"num_object_copies":189672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63224,"num_write_kb":4046336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12265586688,"data_stored":12265586688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63224,"ondisk_log_size":63224,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28818,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1641,"ondisk_log_size":1641,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234485,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1894140,"kb_used_data":1028724,"kb_used_omap":1,"kb_used_meta":865406,"kb_avail":91851012,"statfs":{"total":95995035648,"available":94055436288,"internally_reserved":0,"allocated":1053413376,"data_stored":1051249509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":886176203},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496123,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2573752,"kb_used_data":1788720,"kb_used_omap":1,"kb_used_meta":785022,"kb_avail":91171400,"statfs":{"total":95995035648,"available":93359513600,"internally_reserved":0,"allocated":1831649280,"data_stored":1829489713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":803862983},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569404022,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1760872,"kb_used_data":1199776,"kb_used_omap":7,"kb_used_meta":561080,"kb_avail":91984280,"statfs":{"total":95995035648,"available":94191902720,"internally_reserved":0,"allocated":1228570624,"data_stored":1226415522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":574546840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019396,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2772284,"kb_used_data":2037300,"kb_used_omap":1,"kb_used_meta":734974,"kb_avail":90972868,"statfs":{"total":95995035648,"available":93156216832,"internally_reserved":0,"allocated":2086195200,"data_stored":2084031594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":752613830},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435579,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2149728,"kb_used_data":1129560,"kb_used_omap":5,"kb_used_meta":1020154,"kb_avail":91595424,"statfs":{"total":95995035648,"available":93793714176,"internally_reserved":0,"allocated":1156669440,"data_stored":1154521961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":1044637736},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599107,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2411692,"kb_used_data":1563684,"kb_used_omap":7,"kb_used_meta":847992,"kb_avail":91333460,"statfs":{"total":95995035648,"available":93525463040,"internally_reserved":0,"allocated":1601212416,"data_stored":1599061156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":868344269},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762633,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1546136,"kb_used_data":720336,"kb_used_omap":7,"kb_used_meta":825784,"kb_avail":92199016,"statfs":{"total":95995035648,"available":94411792384,"internally_reserved":0,"allocated":737624064,"data_stored":735483889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":845603069},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":389,"seq":1670742278196,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2610736,"kb_used_data":2442276,"kb_used_omap":12,"kb_used_meta":168435,"kb_avail":91134416,"statfs":{"total":95995035648,"available":93321641984,"internally_reserved":0,"allocated":2500890624,"data_stored":2498739116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12595,"internal_metadata":172478157},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11001,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2512912384,"data_stored":2512912384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":738263040,"data_stored":738263040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1610022912,"data_stored":1610022912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1173684224,"data_stored":1173684224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2098069504,"data_stored":2098069504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1230962688,"data_stored":1230962688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1839267840,"data_stored":1839267840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1062404096,"data_stored":1062404096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:30:08.369 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:30:08.369 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pg_num 21 2024-09-21T10:30:08.394 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:08.441 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:08.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:08.516 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:08.544 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:08.588 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:08.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:08.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:08.690 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:08.728 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:08.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:08.829 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:08.873 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:08.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:08.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:08.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:09.019 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:09.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:09.121 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:09.160 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:09.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:09.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 38 16 70762 70746 116.334 84.9375 0.00821361 0.0085903 2024-09-21T10:30:09.262 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:09.295 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:09.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:09.397 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:09.434 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:09.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:09.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:09.536 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:09.577 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:09.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:09.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:09.678 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:09.727 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:09.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:09.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:09.828 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:09.858 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:30:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:30:09] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.51.0" 2024-09-21T10:30:09.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:09.866 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:09.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:09.967 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:10.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:10.005 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:10.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:10.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:10.143 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:10.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:10.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:10.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 39 16 72161 72145 115.592 87.4375 0.0112494 0.00864663 2024-09-21T10:30:10.186 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:09 smithi032 ceph-mon[105748]: pgmap v1355: 99 pgs: 99 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:30:10.186 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:09 smithi032 ceph-mon[101299]: pgmap v1355: 99 pgs: 99 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:30:10.204 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:09 smithi045 ceph-mon[95976]: pgmap v1355: 99 pgs: 99 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:30:10.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:10.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:10.280 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:10.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:10.380 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:10.412 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:10.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:10.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:10.503 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:30:10.503 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:30:10.503 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:30:10.503 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:30:10.503 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:30:10.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:10.551 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:10.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:10.653 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:10.691 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:10.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:10.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:10.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:10.825 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:10.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:10.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:10.926 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:10.961 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:10.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:10.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:10.977 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:11.051 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:30:11.051 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:30:11.063 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:11.089 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:30:11.089 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:30:11.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:11.100 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:11.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:11.125 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:30:11.125 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:30:11.172 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:30:11.172 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:30:11.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:30:11.184348+0000 min lat: 0.00102285 max lat: 0.278869 avg lat: 0.00871508 2024-09-21T10:30:11.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:30:11.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 40 16 73422 73406 114.673 78.8125 0.00797528 0.00871508 2024-09-21T10:30:11.201 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:11.218 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:30:11.218 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:30:11.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:11.243 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:11.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:11.253 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:30:11.253 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:30:11.280 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:30:11.290 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:30:11.291 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:30:11.329 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:30:11.330 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:30:11.344 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:11.390 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:11.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:11.491 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:11.537 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:11.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:11.639 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:11.645 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:11.674 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:11.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:11.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:11.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:11.815 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:11.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:11.916 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:11.954 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:30:11.954 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:11.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:11.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:11 smithi045 ceph-mon[95976]: pgmap v1356: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-21T10:30:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:11 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1026516652' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:12.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:11 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3619413106' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:30:12.055 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:12.084 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:12.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:12.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:12.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 41 16 74644 74628 113.739 76.375 0.00564126 0.00878603 2024-09-21T10:30:12.185 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:11 smithi032 ceph-mon[101299]: pgmap v1356: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-21T10:30:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:11 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1026516652' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:11 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3619413106' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:30:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:11 smithi032 ceph-mon[105748]: pgmap v1356: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-21T10:30:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:11 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1026516652' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:12.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:11 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3619413106' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:30:12.208 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:12.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:12.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:12.309 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:12.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:12.342 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:12.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:12.443 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:12.465 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:12.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:12.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:12.567 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:12.603 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:12.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:12.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:12.741 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:12.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:12.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:12.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:12.875 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:12.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:12.976 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:13.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:13.017 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:13.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:13.118 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:13.140 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:12 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:30:13.140 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:12 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:30:13.140 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:12 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3619413106' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:30:13.140 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:12 smithi045 ceph-mon[95976]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T10:30:13.164 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:13.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:13.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:13.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 42 16 75793 75777 112.741 71.8125 0.00832057 0.00886466 2024-09-21T10:30:13.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:13.308 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:13.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:13.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:13.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:13.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:13.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:12 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:30:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:12 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:30:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:12 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3619413106' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:30:13.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:12 smithi032 ceph-mon[105748]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T10:30:13.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:12 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:30:13.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:12 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:30:13.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:12 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3619413106' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:30:13.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:12 smithi032 ceph-mon[101299]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T10:30:13.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:13.599 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:13.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:13.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:13.616 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:30:13.616 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:30:13.702 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:13.729 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:13.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:13.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:13.831 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:13.862 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:13.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:13.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:13.964 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:14.004 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:14.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:14.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:14.105 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:14.143 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:14.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:14.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:14.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 43 16 76959 76943 111.813 72.875 0.0128184 0.00893805 2024-09-21T10:30:14.245 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:14.268 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:14.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:14.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:13 smithi045 ceph-mon[95976]: pgmap v1358: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-21T10:30:14.370 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:14.413 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:14.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:13 smithi032 ceph-mon[105748]: pgmap v1358: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-21T10:30:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:13 smithi032 ceph-mon[101299]: pgmap v1358: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-21T10:30:14.514 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:14.554 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:14.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:14.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:14.656 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:14.693 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:14.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:14.795 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:14.835 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:14.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:14.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:14.937 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:14.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:30:14.985 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:14.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:15.087 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:15.129 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:15.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:15.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:15.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 44 16 78233 78217 111.082 79.625 0.0150111 0.00899655 2024-09-21T10:30:15.231 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:15.269 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:15.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:15.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:15.358 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:15.371 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:15.407 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:15.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:15.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:15.509 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:15.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:15.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:15.652 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:15.692 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:15.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:15.793 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:15.840 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:15.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:15.941 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:15.971 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:15.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:15.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:16.072 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:16.116 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:16.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:16.185 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 45 16 79483 79467 110.349 78.125 0.00834358 0.00905726 2024-09-21T10:30:16.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:15 smithi032 ceph-mon[101299]: pgmap v1359: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:30:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:15 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:30:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:15 smithi032 ceph-mon[105748]: pgmap v1359: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:30:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:15 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:30:16.218 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:16.253 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:16.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:16.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:15 smithi045 ceph-mon[95976]: pgmap v1359: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:30:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:15 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:30:16.354 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:16.395 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:16.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:16.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:16.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:16.530 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:16.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:16.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:16.632 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:16.667 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:16.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:16.768 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:16.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:16.819 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:16.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:16.919 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:16.966 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:16.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:16.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:17.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:30:17.068 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:17.129 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:17.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:17.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:17.186 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 46 16 80691 80675 109.59 75.5 0.0161066 0.00911825 2024-09-21T10:30:17.231 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:17.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:17.283 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:17.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:17.384 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:17.419 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:17.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:17.488 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:30:17.488 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:30:17.519 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:17.550 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:17.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:17.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:17.686 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:17.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:17.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:17.787 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:17.813 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:17.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:17.849 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:17.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:17.950 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:17.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:30:17.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:17.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:17.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:18.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:18.111 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:18.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:18.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:18.138 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:17 smithi045 ceph-mon[95976]: pgmap v1360: 99 pgs: 99 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:30:18.176 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:30:18.186 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 47 16 81809 81793 108.745 69.875 0.0186607 0.0091905 2024-09-21T10:30:18.214 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:18.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:18.239 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:18.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:18.340 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:18.362 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:18.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:18.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:18.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:17 smithi032 ceph-mon[101299]: pgmap v1360: 99 pgs: 99 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:30:18.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:17 smithi032 ceph-mon[105748]: pgmap v1360: 99 pgs: 99 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:30:18.464 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:18.525 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:18.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:18.627 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:18.667 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:18.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:18.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:18.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:18.804 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:18.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:18.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:18.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:18.936 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:18.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:18.949 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:19.005 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 1 pg_num to 21 2024-09-21T10:30:19.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:19.085 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:19.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:19.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:19.186 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 48 16 82758 82742 107.716 59.3125 0.0112075 0.00927328 2024-09-21T10:30:19.188 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:19.218 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:19.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:19.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:19.262 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/478760724' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:30:19.262 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:19 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:30:19.262 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:19.262 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:19.320 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:19.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:19.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:19.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:19.455 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/478760724' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:30:19.455 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:19 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:30:19.455 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:19.455 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:19.455 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/478760724' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:30:19.455 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:19 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:30:19.455 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:19.455 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:19.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:19.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:19.502 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:19.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:19.603 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:19.636 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:19.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:19.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:19.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:30:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:30:19] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.51.0" 2024-09-21T10:30:19.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:19.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:19.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:19.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:19.879 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:19.910 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:19.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:20.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:20.049 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:20.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:20.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:20.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:20.186 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 49 16 83930 83914 107.012 73.25 0.0181214 0.00933978 2024-09-21T10:30:20.190 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:20.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:20.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:20 smithi032 ceph-mon[101299]: pgmap v1361: 99 pgs: 99 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:30:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:20 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-21T10:30:20.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:20 smithi032 ceph-mon[101299]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T10:30:20.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:20 smithi032 ceph-mon[105748]: pgmap v1361: 99 pgs: 99 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:30:20.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:20 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-21T10:30:20.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:20 smithi032 ceph-mon[105748]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T10:30:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:20 smithi045 ceph-mon[95976]: pgmap v1361: 99 pgs: 99 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:30:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:20 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-21T10:30:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:20 smithi045 ceph-mon[95976]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T10:30:20.292 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:20.325 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:20.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:20.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:20.454 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:20.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:20.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:20.556 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:20.589 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:20.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:20.691 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:20.715 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:20.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:20.817 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:20.863 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:20.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:20.965 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:21.002 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:21.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:21.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:21.106 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:21.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:21.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:21.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:21.186 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 50 16 85088 85072 106.319 72.375 0.0129935 0.00940045 2024-09-21T10:30:21.248 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:21.284 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:21.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-21T10:30:21.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:21.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:21.289 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:21.386 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:21.409 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-21T10:30:21.409 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-21T10:30:21.429 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:21.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:21.531 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:21.559 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:21.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:21.660 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:21.702 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:21.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:21.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:21.804 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:21.834 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:21.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:21.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:21.936 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:21.978 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:21.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:21.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:22.080 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:22.102 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:22 smithi032 ceph-mon[101299]: pgmap v1363: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:30:22.102 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:22 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-21T10:30:22.102 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:22 smithi032 ceph-mon[101299]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T10:30:22.102 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:22 smithi032 ceph-mon[105748]: pgmap v1363: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:30:22.102 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:22 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-21T10:30:22.103 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:22 smithi032 ceph-mon[105748]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T10:30:22.111 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:22.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:22.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 51 16 86165 86149 105.554 67.3125 0.0127029 0.00946884 2024-09-21T10:30:22.212 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:22.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:22.246 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:22.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:22 smithi045 ceph-mon[95976]: pgmap v1363: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:30:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:22 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-21T10:30:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:22 smithi045 ceph-mon[95976]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T10:30:22.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:22.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:22.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:22.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:22.480 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:22.502 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:22.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:22.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:22.638 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:22.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:22.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:22.740 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:22.784 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:22.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:22.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:22.886 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:22.923 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:22.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:22.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:23.024 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:23.059 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:23.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:23.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:23.161 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:23.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 52 16 87200 87184 104.768 64.6875 0.01199 0.00954006 2024-09-21T10:30:23.200 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:23.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:23.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:23 smithi032 ceph-mon[101299]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T10:30:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:23 smithi032 ceph-mon[101299]: 1.1 scrub starts 2024-09-21T10:30:23.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:23 smithi032 ceph-mon[101299]: 1.1 scrub ok 2024-09-21T10:30:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:23 smithi032 ceph-mon[105748]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T10:30:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:23 smithi032 ceph-mon[105748]: 1.1 scrub starts 2024-09-21T10:30:23.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:23 smithi032 ceph-mon[105748]: 1.1 scrub ok 2024-09-21T10:30:23.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:23 smithi045 ceph-mon[95976]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T10:30:23.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:23 smithi045 ceph-mon[95976]: 1.1 scrub starts 2024-09-21T10:30:23.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:23 smithi045 ceph-mon[95976]: 1.1 scrub ok 2024-09-21T10:30:23.302 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:23.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:23.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:23.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:23.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:23.494 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:23.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:23.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:23.596 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:23.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:23.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:23.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:23.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:23.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:23.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:23.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:23.871 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:23.907 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:23.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:23.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:24.009 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:24.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 53 16 88383 88367 104.186 73.9375 0.0182124 0.00959289 2024-09-21T10:30:24.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:24.211 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:24.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:24 smithi045 ceph-mon[95976]: 1.5 scrub starts 2024-09-21T10:30:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:24 smithi045 ceph-mon[95976]: 1.5 scrub ok 2024-09-21T10:30:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:24 smithi045 ceph-mon[95976]: 1.3 scrub starts 2024-09-21T10:30:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:24 smithi045 ceph-mon[95976]: 1.3 scrub ok 2024-09-21T10:30:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:24 smithi045 ceph-mon[95976]: pgmap v1366: 109 pgs: 2 peering, 10 unknown, 97 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:30:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:24 smithi045 ceph-mon[95976]: 1.4 scrub starts 2024-09-21T10:30:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:24 smithi045 ceph-mon[95976]: 1.4 scrub ok 2024-09-21T10:30:24.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:24.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[105748]: 1.5 scrub starts 2024-09-21T10:30:24.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[105748]: 1.5 scrub ok 2024-09-21T10:30:24.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[105748]: 1.3 scrub starts 2024-09-21T10:30:24.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[105748]: 1.3 scrub ok 2024-09-21T10:30:24.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[105748]: pgmap v1366: 109 pgs: 2 peering, 10 unknown, 97 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:30:24.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[105748]: 1.4 scrub starts 2024-09-21T10:30:24.324 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[105748]: 1.4 scrub ok 2024-09-21T10:30:24.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[101299]: 1.5 scrub starts 2024-09-21T10:30:24.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[101299]: 1.5 scrub ok 2024-09-21T10:30:24.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[101299]: 1.3 scrub starts 2024-09-21T10:30:24.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[101299]: 1.3 scrub ok 2024-09-21T10:30:24.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[101299]: pgmap v1366: 109 pgs: 2 peering, 10 unknown, 97 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:30:24.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[101299]: 1.4 scrub starts 2024-09-21T10:30:24.325 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:24 smithi032 ceph-mon[101299]: 1.4 scrub ok 2024-09-21T10:30:24.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:24.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:24.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:24.451 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:24.492 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:24.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:24.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:24.594 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:24.619 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:24.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:24.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:24.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:24.739 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:30:24.739 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:30:24.739 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:30:24.739 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:30:24.740 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:30:24.754 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:24.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:24.856 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:24.895 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:24.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:24.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:24.996 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:25.032 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:25.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:25.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:25.134 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:25.163 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:25.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:25.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:25.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 54 16 89325 89309 103.347 58.875 0.00378126 0.00967041 2024-09-21T10:30:25.265 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:25.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:25.305 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:25.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:25.393 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:25 smithi045 ceph-mon[95976]: 1.14 scrub starts 2024-09-21T10:30:25.393 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:25 smithi045 ceph-mon[95976]: 1.14 scrub ok 2024-09-21T10:30:25.393 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/336136788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:25.406 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:25 smithi032 ceph-mon[105748]: 1.14 scrub starts 2024-09-21T10:30:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:25 smithi032 ceph-mon[105748]: 1.14 scrub ok 2024-09-21T10:30:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/336136788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:25.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:25 smithi032 ceph-mon[101299]: 1.14 scrub starts 2024-09-21T10:30:25.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:25 smithi032 ceph-mon[101299]: 1.14 scrub ok 2024-09-21T10:30:25.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/336136788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:25.463 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:30:25.464 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:30:25.464 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:30:25.464 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:30:25.464 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:30:25.470 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:25.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:25.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:25.572 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:25.596 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:25.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:25.698 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:25.730 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:25.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:25.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:25.733 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 1, 0, 4] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 6, 7, 5, 0] 2024-09-21T10:30:25.733 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:30:25.733 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [2, 3, 5, 6, 7, 1, 0, 4] 2024-09-21T10:30:25.733 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd out 1 2024-09-21T10:30:25.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:25.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:25.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:25.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:25.986 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:26.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:26.031 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:26.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:26.132 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:26.173 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:26.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:26.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:26.187 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 55 16 90350 90334 102.632 64.0625 0.0114151 0.00973813 2024-09-21T10:30:26.196 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:30:26.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:26 smithi045 ceph-mon[95976]: 1.d scrub starts 2024-09-21T10:30:26.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:26 smithi045 ceph-mon[95976]: 1.d scrub ok 2024-09-21T10:30:26.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:26 smithi045 ceph-mon[95976]: 1.b scrub starts 2024-09-21T10:30:26.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:26 smithi045 ceph-mon[95976]: 1.b scrub ok 2024-09-21T10:30:26.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:26 smithi045 ceph-mon[95976]: pgmap v1367: 109 pgs: 2 peering, 5 unknown, 102 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-21T10:30:26.243 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:26 smithi045 ceph-mon[95976]: 1.c scrub starts 2024-09-21T10:30:26.244 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:26 smithi045 ceph-mon[95976]: 1.c scrub ok 2024-09-21T10:30:26.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:26.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:26.309 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:26.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:26.410 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:26.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[101299]: 1.d scrub starts 2024-09-21T10:30:26.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[101299]: 1.d scrub ok 2024-09-21T10:30:26.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[101299]: 1.b scrub starts 2024-09-21T10:30:26.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[101299]: 1.b scrub ok 2024-09-21T10:30:26.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[101299]: pgmap v1367: 109 pgs: 2 peering, 5 unknown, 102 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-21T10:30:26.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[101299]: 1.c scrub starts 2024-09-21T10:30:26.441 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[101299]: 1.c scrub ok 2024-09-21T10:30:26.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[105748]: 1.d scrub starts 2024-09-21T10:30:26.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[105748]: 1.d scrub ok 2024-09-21T10:30:26.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[105748]: 1.b scrub starts 2024-09-21T10:30:26.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[105748]: 1.b scrub ok 2024-09-21T10:30:26.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[105748]: pgmap v1367: 109 pgs: 2 peering, 5 unknown, 102 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-21T10:30:26.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[105748]: 1.c scrub starts 2024-09-21T10:30:26.442 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:26 smithi032 ceph-mon[105748]: 1.c scrub ok 2024-09-21T10:30:26.456 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:26.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:26.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:26.604 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:26.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:26.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:26.706 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:26.738 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:26.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:26.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:26.839 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:26.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:26.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:26.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:26.994 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:27.003 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:27.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:27.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:27.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:27.140 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:27.173 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:27.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:27.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:27.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 56 16 91273 91257 101.829 57.6875 0.0183235 0.00981462 2024-09-21T10:30:27.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:27.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:27.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:27.336 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[105748]: 1.2 scrub starts 2024-09-21T10:30:27.336 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[105748]: 1.2 scrub ok 2024-09-21T10:30:27.336 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[105748]: 1.11 scrub starts 2024-09-21T10:30:27.336 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[105748]: 1.11 scrub ok 2024-09-21T10:30:27.336 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:30:27.336 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[101299]: 1.2 scrub starts 2024-09-21T10:30:27.336 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[101299]: 1.2 scrub ok 2024-09-21T10:30:27.337 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[101299]: 1.11 scrub starts 2024-09-21T10:30:27.337 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[101299]: 1.11 scrub ok 2024-09-21T10:30:27.337 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:27 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:30:27.340 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:27 smithi045 ceph-mon[95976]: 1.2 scrub starts 2024-09-21T10:30:27.340 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:27 smithi045 ceph-mon[95976]: 1.2 scrub ok 2024-09-21T10:30:27.340 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:27 smithi045 ceph-mon[95976]: 1.11 scrub starts 2024-09-21T10:30:27.340 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:27 smithi045 ceph-mon[95976]: 1.11 scrub ok 2024-09-21T10:30:27.340 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:27 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:30:27.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:27.466 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:27.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:27.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:27.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:27.601 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:27.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:27.703 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:27.737 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:27.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:27.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:27.839 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:27.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:27.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:27.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:27.981 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:28.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:28.034 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:28.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:28.093 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:30:28.135 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:28.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:28.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:28.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:28.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 57 16 92280 92264 101.147 62.9375 0.0162922 0.00988149 2024-09-21T10:30:28.275 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:28.311 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:28.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:28.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:28.413 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:28.453 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:28.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:28.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[101299]: 1.13 scrub starts 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[101299]: 1.13 scrub ok 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[101299]: pgmap v1368: 109 pgs: 109 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[101299]: 1.6 scrub starts 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[101299]: 1.6 scrub ok 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[101299]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1507614352' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[105748]: 1.13 scrub starts 2024-09-21T10:30:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[105748]: 1.13 scrub ok 2024-09-21T10:30:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[105748]: pgmap v1368: 109 pgs: 109 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-21T10:30:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[105748]: 1.6 scrub starts 2024-09-21T10:30:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[105748]: 1.6 scrub ok 2024-09-21T10:30:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:30:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[105748]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T10:30:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1507614352' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:30:28.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:28 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:30:28.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:30:28.519 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:28 smithi045 ceph-mon[95976]: 1.13 scrub starts 2024-09-21T10:30:28.519 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:28 smithi045 ceph-mon[95976]: 1.13 scrub ok 2024-09-21T10:30:28.519 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:28 smithi045 ceph-mon[95976]: pgmap v1368: 109 pgs: 109 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-21T10:30:28.519 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:28 smithi045 ceph-mon[95976]: 1.6 scrub starts 2024-09-21T10:30:28.519 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:28 smithi045 ceph-mon[95976]: 1.6 scrub ok 2024-09-21T10:30:28.519 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:28 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:30:28.519 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:28 smithi045 ceph-mon[95976]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T10:30:28.519 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:28 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1507614352' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:30:28.519 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:28 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:30:28.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:28.583 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:28.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:28.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:28.685 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:28.720 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:28.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:28.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:28.822 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:28.860 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:28.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:28.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:28.962 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:28.995 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:28.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:28.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:29.097 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:29.145 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:29.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:29.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:29.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 58 16 93385 93369 100.594 69.0625 0.00639402 0.00993625 2024-09-21T10:30:29.247 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:29.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:29.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:29.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:29.382 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:29 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:29.382 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:29 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:30:29.382 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:29 smithi032 ceph-mon[101299]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T10:30:29.382 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:29 smithi032 ceph-mon[101299]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T10:30:29.382 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:29 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:29.382 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:29 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:30:29.382 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:29 smithi032 ceph-mon[105748]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T10:30:29.382 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:29 smithi032 ceph-mon[105748]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T10:30:29.385 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:29.431 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:29.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:29.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:29 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:29.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:29 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:30:29.439 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:29 smithi045 ceph-mon[95976]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T10:30:29.440 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:29 smithi045 ceph-mon[95976]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T10:30:29.532 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:29.571 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:29.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:29.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:29.673 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:29.706 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:30:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:30:29] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.51.0" 2024-09-21T10:30:29.714 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:29.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:29.816 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:29.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:29.863 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:29.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:29.964 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:30.010 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:30.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:30.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:30.112 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:30.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:30.163 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:30.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:30.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 59 16 94543 94527 100.115 72.375 0.0126326 0.00998295 2024-09-21T10:30:30.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:30.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:30 smithi045 ceph-mon[95976]: pgmap v1371: 109 pgs: 1 unknown, 108 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 5.7 KiB/s rd, 57 MiB/s wr, 919 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-21T10:30:30.267 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:30 smithi045 ceph-mon[95976]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T10:30:30.313 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:30.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:30.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:30.399 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:30 smithi032 ceph-mon[105748]: pgmap v1371: 109 pgs: 1 unknown, 108 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 5.7 KiB/s rd, 57 MiB/s wr, 919 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-21T10:30:30.399 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:30 smithi032 ceph-mon[105748]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T10:30:30.399 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:30 smithi032 ceph-mon[101299]: pgmap v1371: 109 pgs: 1 unknown, 108 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 5.7 KiB/s rd, 57 MiB/s wr, 919 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-21T10:30:30.400 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:30 smithi032 ceph-mon[101299]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T10:30:30.415 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:30.457 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:30.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:30.558 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:30.604 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:30.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:30.705 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:30.747 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:30.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:30.848 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:30.889 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:30.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:30.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:30.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:31.034 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:31.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:31.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:31.136 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:31.184 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:31.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:30:31.187527+0000 min lat: 0.00102285 max lat: 0.278869 avg lat: 0.0100382 2024-09-21T10:30:31.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:30:31.188 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 60 16 95608 95592 99.5559 66.5625 0.0127964 0.0100382 2024-09-21T10:30:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:31.193 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:31.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:31.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:31.328 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:31.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:31.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:31.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:31.461 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:31.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:31.563 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:31.597 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:31.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:31.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:31.698 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:31.706 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:31 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:30:31.565+0000 7f532c355640 -1 mon.a@0(leader).osd e436 definitely_dead 0 2024-09-21T10:30:31.734 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:31.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:31.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:31.835 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:31.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:142730 9.3 9:caf5744d:::benchmark_data_smithi032_265250_object14928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.656116+0000", 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 248.99644129999999, 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14369104699999999, 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.873 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 142730 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656116+0000", 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656116+0000", 2024-09-21T10:30:31.874 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656115+0000", 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656120+0000", 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6609999999999997e-06 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656131+0000", 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0849000000000001e-05 2024-09-21T10:30:31.875 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656138+0000", 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9199999999999998e-06 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656179+0000", 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1362000000000001e-05 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656278+0000", 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8797999999999998e-05 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.876 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656398+0000", 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120029 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.659952+0000", 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0035538010000000001 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797529+0000", 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13757687900000001 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.877 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799595+0000", 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0020662549999999999 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799761+0000", 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016636099999999999 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799807+0000", 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5318000000000002e-05 2024-09-21T10:30:31.878 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:142731 9.3 9:cf90c21c:::benchmark_data_smithi032_265250_object14929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.660201+0000", 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 248.99235557599999, 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13972269700000001, 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 142731 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.879 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660201+0000", 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660201+0000", 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660199+0000", 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.880 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660206+0000", 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7139999999999996e-06 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660218+0000", 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1960000000000001e-05 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660225+0000", 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7619999999999998e-06 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.881 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660255+0000", 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0051000000000001e-05 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660328+0000", 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3061999999999999e-05 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660405+0000", 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6862999999999999e-05 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663603+0000", 2024-09-21T10:30:31.882 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003198116 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797574+0000", 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.133970802 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799849+0000", 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022749430000000002 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799888+0000", 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8939000000000001e-05 2024-09-21T10:30:31.883 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799924+0000", 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6594999999999998e-05 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:142732 9.9 e394/390 9:92131257:::benchmark_data_smithi032_265250_object14930:head v 394'9008, mlcod=382'2729)", 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.663542+0000", 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 248.98901477300001, 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.134083435, 2024-09-21T10:30:31.884 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663542+0000", 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663542+0000", 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663542+0000", 2024-09-21T10:30:31.885 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663549+0000", 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6310000000000003e-06 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663560+0000", 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0997e-05 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.886 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663566+0000", 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2310000000000001e-06 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663630+0000", 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3108999999999995e-05 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663645+0000", 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5339e-05 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.887 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797594+0000", 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.133949278 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797626+0000", 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1529999999999998e-05 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:142733 9.d 9:bb8717e3:::benchmark_data_smithi032_265250_object14931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.663640+0000", 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 248.98891717199999, 2024-09-21T10:30:31.888 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14898618299999999, 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 142733 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663640+0000", 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.889 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663640+0000", 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663639+0000", 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663644+0000", 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4620000000000004e-06 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663653+0000", 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0729999999999994e-06 2024-09-21T10:30:31.890 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663659+0000", 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1630000000000001e-06 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663691+0000", 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1903000000000003e-05 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663789+0000", 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.7422000000000003e-05 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.891 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797500+0000", 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13371169299999999 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.811231+0000", 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013731220000000001 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.811287+0000", 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5899999999999997e-05 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.892 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.812552+0000", 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0012642 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.812593+0000", 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1242999999999998e-05 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.812626+0000", 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3198999999999998e-05 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.893 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:142730 9.3 e394/378 ondisk, result = 0)", 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.665234+0000", 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 248.987322917, 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13458094600000001, 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.665234+0000", 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.894 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.665234+0000", 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.665233+0000", 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.665239+0000", 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1770000000000003e-06 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.895 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797638+0000", 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.132399348 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797645+0000", 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3659999999999997e-06 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799536+0000", 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0018909129999999999 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.896 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799569+0000", 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3500000000000001e-05 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799815+0000", 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024582199999999998 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:142737 9.7 e394/378 9:e247ab7e:::benchmark_data_smithi032_265250_object14935:head v 394'8911, mlcod=394'8908)", 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.668401+0000", 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 248.98415566, 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13380583300000001, 2024-09-21T10:30:31.897 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.668401+0000", 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.668401+0000", 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.898 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.668400+0000", 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.668410+0000", 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.9620000000000005e-06 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797417+0000", 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.129007388 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797432+0000", 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4533e-05 2024-09-21T10:30:31.899 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797486+0000", 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4326000000000001e-05 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797512+0000", 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5979999999999999e-05 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.802141+0000", 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.004628521 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.900 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.802207+0000", 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6471999999999996e-05 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:171427 9.d 9:b369ed47:::benchmark_data_smithi032_265250_object43625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:58.579006+0000", 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 213.07355084700001, 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.164263189, 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:30:31.901 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 171427 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579006+0000", 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579006+0000", 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.902 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579005+0000", 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579011+0000", 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9330000000000004e-06 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579028+0000", 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6518999999999999e-05 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579036+0000", 2024-09-21T10:30:31.903 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6869999999999994e-06 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579113+0000", 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6953000000000007e-05 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579240+0000", 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012727699999999999 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579369+0000", 2024-09-21T10:30:31.904 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000129498 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.583851+0000", 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0044817190000000003 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.583859+0000", 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4549999999999998e-06 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.743172+0000", 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15931371899999999 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.905 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.743228+0000", 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5472e-05 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.743269+0000", 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1585999999999998e-05 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:171437 9.d 9:b73bf904:::benchmark_data_smithi032_265250_object43635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-09-21T10:30:31.906 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:58.592651+0000", 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 213.05990559200001, 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15252275100000001, 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 171437 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592651+0000", 2024-09-21T10:30:31.907 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592651+0000", 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592650+0000", 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592655+0000", 2024-09-21T10:30:31.908 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7090000000000001e-06 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592663+0000", 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.011e-06 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592668+0000", 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4240000000000003e-06 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592698+0000", 2024-09-21T10:30:31.909 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9977000000000001e-05 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592783+0000", 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5944000000000002e-05 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592863+0000", 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9072000000000005e-05 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.596988+0000", 2024-09-21T10:30:31.910 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0041250569999999997 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.597692+0000", 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00070395799999999997 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745079+0000", 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.147387925 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.911 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745124+0000", 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4453000000000003e-05 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745174+0000", 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0219999999999997e-05 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:171438 9.d 9:b6209aad:::benchmark_data_smithi032_265250_object43636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:58.592704+0000", 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 213.05985271899999, 2024-09-21T10:30:31.912 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15258337299999999, 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 171438 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592704+0000", 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.913 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592704+0000", 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592704+0000", 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592706+0000", 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.272e-06 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.914 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592712+0000", 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3809999999999998e-06 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592715+0000", 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2770000000000001e-06 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.595442+0000", 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0027276430000000001 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.915 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.595510+0000", 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7891000000000004e-05 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.595577+0000", 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6830999999999998e-05 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.597115+0000", 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015379 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.916 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.597720+0000", 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00060456099999999999 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745217+0000", 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14749691500000001 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745256+0000", 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9857000000000001e-05 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.917 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745288+0000", 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1208999999999999e-05 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186530 9.a e404/378 9:5f76c579:::benchmark_data_smithi032_265250_object58728:head v 404'11664, mlcod=404'11661)", 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.943986+0000", 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 193.70857106400001, 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.140619997, 2024-09-21T10:30:31.918 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.943986+0000", 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.919 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.943986+0000", 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.943985+0000", 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.943992+0000", 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4819999999999999e-06 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.944003+0000", 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1539000000000001e-05 2024-09-21T10:30:31.920 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.944010+0000", 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0310000000000001e-06 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.944379+0000", 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000368826 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.944398+0000", 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8862e-05 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.921 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950107+0000", 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.005708879 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084606+0000", 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13449923699999999 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:186535 9.d 9:b402cb59:::benchmark_data_smithi032_265250_object58733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e404)", 2024-09-21T10:30:31.922 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.945797+0000", 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 193.706759554, 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.142660606, 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 186535 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945797+0000", 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.923 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945797+0000", 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945796+0000", 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945802+0000", 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3260000000000001e-06 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.924 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945814+0000", 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2048999999999999e-05 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945821+0000", 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5749999999999997e-06 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945922+0000", 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000100874 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.925 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.946026+0000", 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010377100000000001 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.946127+0000", 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000101897 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084660+0000", 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.138532405 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.926 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084668+0000", 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8029999999999997e-06 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088340+0000", 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003672168 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088412+0000", 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2311999999999995e-05 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.927 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088458+0000", 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5905000000000002e-05 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186534 9.7 e404/378 9:e7392b0d:::benchmark_data_smithi032_265250_object58732:head v 404'11653, mlcod=404'11650)", 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.945845+0000", 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 193.706711539, 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13880384200000001, 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.928 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945845+0000", 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945845+0000", 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945845+0000", 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:31.929 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945849+0000", 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2599999999999999e-06 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945856+0000", 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9670000000000004e-06 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945861+0000", 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7639999999999996e-06 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.930 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945907+0000", 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5587000000000002e-05 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945925+0000", 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8039000000000001e-05 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084615+0000", 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13869034099999999 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.931 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084649+0000", 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4279999999999997e-05 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186536 9.c e404/378 9:3357cb79:::benchmark_data_smithi032_265250_object58734:head v 404'11671, mlcod=404'11669)", 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.947568+0000", 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 193.704989162, 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.140591682, 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.932 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947568+0000", 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947568+0000", 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947567+0000", 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.933 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947575+0000", 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4880000000000007e-06 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947587+0000", 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1748e-05 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947594+0000", 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.391e-06 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.934 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947655+0000", 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0457000000000001e-05 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947675+0000", 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9763e-05 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088122+0000", 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14044704799999999 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088160+0000", 2024-09-21T10:30:31.935 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7774000000000001e-05 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186528 9.5 e404/390 9:a6579e44:::benchmark_data_smithi032_265250_object58726:head v 404'11718, mlcod=385'5737)", 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.950012+0000", 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 193.70254486100001, 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15133627099999999, 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950012+0000", 2024-09-21T10:30:31.936 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950012+0000", 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950011+0000", 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950018+0000", 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4219999999999997e-06 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.937 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950028+0000", 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6889999999999999e-06 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950033+0000", 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8459999999999996e-06 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950149+0000", 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115339 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.938 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950169+0000", 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0564000000000002e-05 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.099276+0000", 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14910671 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.101348+0000", 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002072384 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.939 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186538 9.c e404/378 9:3246b83d:::benchmark_data_smithi032_265250_object58736:head v 404'11672, mlcod=404'11669)", 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.951145+0000", 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 193.70141193000001, 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.144223773, 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.951145+0000", 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.940 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.951145+0000", 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.951144+0000", 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.951151+0000", 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8469999999999998e-06 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.941 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084575+0000", 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13342368399999999 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084583+0000", 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3049999999999993e-06 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084753+0000", 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016996399999999999 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084777+0000", 2024-09-21T10:30:31.942 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4063999999999999e-05 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095335+0000", 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010557715000000001 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095369+0000", 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4022000000000001e-05 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.943 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:186541 9.e 9:72d26f2c:::benchmark_data_smithi032_265250_object58739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e404)", 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.953425+0000", 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 193.69913225799999, 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14240792499999999, 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 186541 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.953425+0000", 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.944 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.953425+0000", 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.953423+0000", 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.953431+0000", 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8909999999999995e-06 2024-09-21T10:30:31.945 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084557+0000", 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13112615699999999 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084573+0000", 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5968e-05 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084620+0000", 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6789000000000002e-05 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.946 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084748+0000", 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012812000000000001 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084879+0000", 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013129 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088729+0000", 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0038496479999999998 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.947 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.091647+0000", 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029184129999999999 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095709+0000", 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0040617040000000002 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095793+0000", 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3704999999999999e-05 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095833+0000", 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0144000000000001e-05 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:186535 9.d e404/378 ondisk, result = 0)", 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.956863+0000", 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 193.69569361000001, 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.12780517699999999, 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.956863+0000", 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.956863+0000", 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.956862+0000", 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.950 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.956871+0000", 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5790000000000004e-06 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084560+0000", 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.12768947999999999 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084571+0000", 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0645000000000001e-05 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.951 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084641+0000", 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0432000000000006e-05 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084652+0000", 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.115e-05 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084669+0000", 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6120000000000002e-05 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.952 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:45866 10.c 10:3b641306:::benchmark_data_smithi032_297285_object45865:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:51.556348+0000", 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 40.096209373000001, 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.18469603100000001, 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 45866 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.953 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556348+0000", 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556348+0000", 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556347+0000", 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556403+0000", 2024-09-21T10:30:31.954 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5930000000000002e-05 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556418+0000", 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5174000000000001e-05 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556424+0000", 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0290000000000002e-06 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.558347+0000", 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0019228330000000001 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.955 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.558514+0000", 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000167446 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.558657+0000", 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014288500000000001 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.562496+0000", 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0038387500000000001 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.956 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.562557+0000", 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0871e-05 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.740917+0000", 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.17835991900000001 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.740994+0000", 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6959000000000006e-05 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.741044+0000", 2024-09-21T10:30:31.957 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9864999999999998e-05 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:45869 10.c 10:39383c1a:::benchmark_data_smithi032_297285_object45868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:51.557825+0000", 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 40.094732239000002, 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.189745101, 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:31.958 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 45869 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557825+0000", 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557825+0000", 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.959 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557823+0000", 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557878+0000", 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4939999999999999e-05 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557893+0000", 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4817999999999999e-05 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.960 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557900+0000", 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9460000000000001e-06 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.560539+0000", 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002638376 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.560670+0000", 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013103 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.560790+0000", 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120327 2024-09-21T10:30:31.961 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.562501+0000", 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0017107489999999999 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.563165+0000", 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00066432600000000002 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.747434+0000", 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.184268722 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.962 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.747519+0000", 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5310999999999997e-05 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.747570+0000", 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0856999999999998e-05 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:45896 10.c 10:3be2d8d5:::benchmark_data_smithi032_297285_object45895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-09-21T10:30:31.963 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:51.577908+0000", 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 40.074649473999997, 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.17023393000000001, 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 45896 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577908+0000", 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.964 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577908+0000", 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577906+0000", 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577958+0000", 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2077000000000003e-05 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.965 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577970+0000", 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2072000000000001e-05 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577977+0000", 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2169999999999997e-06 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.578026+0000", 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8439000000000002e-05 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.966 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.578170+0000", 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014389700000000001 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.578318+0000", 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014834200000000001 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.579725+0000", 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0014065900000000001 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.579957+0000", 2024-09-21T10:30:31.967 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00023208699999999999 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.748060+0000", 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16810291899999999 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.748108+0000", 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7919e-05 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.748141+0000", 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3880000000000001e-05 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.968 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.969 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.969 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:31.969 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:31.969 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:30:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:31.970 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:31.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:32.071 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:32.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:32.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:32.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:32.129 INFO:teuthology.orchestra.run.smithi032.stderr:marked out osd.1. 2024-09-21T10:30:32.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 61 16 96640 96624 98.981 64.5 0.0119992 0.0100968 2024-09-21T10:30:32.212 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:32.237 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:32.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:32.339 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:32.361 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:32.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:32.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[101299]: pgmap v1374: 109 pgs: 4 active+undersized+remapped, 1 unknown, 104 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 19 KiB/s rd, 64 MiB/s wr, 1.03k op/s; 145 KiB/s, 0 objects/s recovering 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[101299]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4167574127' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[101299]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[105748]: pgmap v1374: 109 pgs: 4 active+undersized+remapped, 1 unknown, 104 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 19 KiB/s rd, 64 MiB/s wr, 1.03k op/s; 145 KiB/s, 0 objects/s recovering 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[105748]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4167574127' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:30:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:32 smithi032 ceph-mon[105748]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:30:32.463 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:32.484 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:32.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:32.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:32 smithi045 ceph-mon[95976]: pgmap v1374: 109 pgs: 4 active+undersized+remapped, 1 unknown, 104 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 19 KiB/s rd, 64 MiB/s wr, 1.03k op/s; 145 KiB/s, 0 objects/s recovering 2024-09-21T10:30:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:32 smithi045 ceph-mon[95976]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T10:30:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:32 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4167574127' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:30:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:32 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:30:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:32 smithi045 ceph-mon[95976]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:30:32.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:32.608 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:32.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:32.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:32.709 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:32.732 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:32.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:32.833 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:32.857 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:32.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:32.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:32.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:32.980 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:32.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:33.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:33.105 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:33.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:33.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 62 16 96685 96669 97.43 2.8125 0.00329821 0.010095 2024-09-21T10:30:33.205 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:33.230 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:33.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:33.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:33.290 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:30:33.332 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:33.356 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:33.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:33.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:33.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:33 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:30:33.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:33 smithi032 ceph-mon[101299]: osdmap e437: 8 total, 8 up, 7 in 2024-09-21T10:30:33.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:33 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:30:33.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:33 smithi032 ceph-mon[105748]: osdmap e437: 8 total, 8 up, 7 in 2024-09-21T10:30:33.457 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:33.480 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:33.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:33.513 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:33 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:30:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:33 smithi045 ceph-mon[95976]: osdmap e437: 8 total, 8 up, 7 in 2024-09-21T10:30:33.582 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:33.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:33.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:33.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:33.706 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:33.730 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:33.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:33.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:33.831 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:33.854 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:33.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:33.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:33.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:33.978 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:33.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:34.079 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:34.104 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:34.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:34.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 63 16 96685 96669 95.8836 0 - 0.010095 2024-09-21T10:30:34.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[105748]: pgmap v1377: 109 pgs: 4 peering, 4 active+undersized+remapped, 2 unknown, 99 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 28 KiB/s rd, 80 MiB/s wr, 1.30k op/s; 178 KiB/s, 0 objects/s recovering 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[105748]: osdmap e438: 8 total, 8 up, 7 in 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[101299]: pgmap v1377: 109 pgs: 4 peering, 4 active+undersized+remapped, 2 unknown, 99 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 28 KiB/s rd, 80 MiB/s wr, 1.30k op/s; 178 KiB/s, 0 objects/s recovering 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[101299]: osdmap e438: 8 total, 8 up, 7 in 2024-09-21T10:30:34.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:34.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:34.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:34.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:34.245 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:34.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:34.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:34 smithi045 ceph-mon[95976]: pgmap v1377: 109 pgs: 4 peering, 4 active+undersized+remapped, 2 unknown, 99 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 28 KiB/s rd, 80 MiB/s wr, 1.30k op/s; 178 KiB/s, 0 objects/s recovering 2024-09-21T10:30:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:34 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-09-21T10:30:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:34 smithi045 ceph-mon[95976]: osdmap e438: 8 total, 8 up, 7 in 2024-09-21T10:30:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:34.347 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:34.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:34.381 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:34.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:34.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:34.510 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:34.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:34.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:34.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:34.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:34.643 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:34.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:34.744 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:34.816 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:34.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:34.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:34.917 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:34.943 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:34.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:34.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:35.044 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:35.074 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:35.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:35.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:35.175 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:35.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 64 16 97309 97293 94.9948 19.5 0.0182449 0.0105207 2024-09-21T10:30:35.219 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:35.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:35.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:35.268 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:35 smithi045 ceph-mon[95976]: osdmap e439: 8 total, 8 up, 7 in 2024-09-21T10:30:35.322 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:35.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:35.365 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:35.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.36234.0:97481 10.9 e440/419)", 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:35.426680+0000", 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.016762183, 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.016795781999999999, 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:35.447 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.426680+0000", 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.426680+0000", 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.426679+0000", 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:35.448 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.426685+0000", 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0639999999999999e-06 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.426699+0000", 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4049999999999999e-05 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.426707+0000", 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1849999999999998e-06 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.449 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.36234.0:97481 10.9 e440/419)", 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:35.427541+0000", 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.015901423000000001, 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015994702999999999, 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "queued for pg", 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.427541+0000", 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:35.450 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.427541+0000", 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.427540+0000", 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.427546+0000", 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7049999999999996e-06 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.451 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.427561+0000", 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5319e-05 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.427568+0000", 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9090000000000003e-06 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:97485 10.a e440/419 10:527c491f:::benchmark_data_smithi032_297285_object97484:head v 440'6101, mlcod=440'6099)", 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:35.434225+0000", 2024-09-21T10:30:35.452 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.009217645, 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0093605860000000006, 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.434225+0000", 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.434225+0000", 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.453 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.434224+0000", 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.434253+0000", 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9535000000000001e-05 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.434271+0000", 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.7555999999999999e-05 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.454 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.434278+0000", 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1899999999999998e-06 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.434346+0000", 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7510999999999998e-05 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.434368+0000", 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2081000000000001e-05 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:35.455 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.438893+0000", 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0045255440000000003 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.36234.0:97495 10.3 e440/419 10:cd95f283:::benchmark_data_smithi032_297285_object97494:head v 440'6166, mlcod=440'6164)", 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:35.435204+0000", 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.0082384710000000007, 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0084455799999999994, 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:35.456 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.435204+0000", 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.435204+0000", 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.435203+0000", 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.457 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.435268+0000", 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4799000000000001e-05 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.435283+0000", 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5051999999999999e-05 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.435290+0000", 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5499999999999997e-06 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.435343+0000", 2024-09-21T10:30:35.458 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2485999999999999e-05 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.435363+0000", 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0373999999999998e-05 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:97481 10.9 10:93c91502:::benchmark_data_smithi032_297285_object97480:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:35.417833+0000", 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.025609726999999999, 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.025884771000000001, 2024-09-21T10:30:35.459 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 97481 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.417833+0000", 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:35.460 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.417833+0000", 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.417831+0000", 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.417886+0000", 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5481e-05 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.417905+0000", 2024-09-21T10:30:35.461 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8802000000000001e-05 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.417915+0000", 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5440000000000004e-06 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.420645+0000", 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0027302870000000001 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:35.462 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.420793+0000", 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014789499999999999 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.420940+0000", 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014767200000000001 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:97484 10.9 10:9041745e:::benchmark_data_smithi032_297285_object97483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2024-09-21T10:30:35.463 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:35.424298+0000", 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 0.019144952, 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.019536296000000002, 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 97484 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.424298+0000", 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.464 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.424298+0000", 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.424296+0000", 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.424385+0000", 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9019999999999998e-05 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.465 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.424405+0000", 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0055999999999999e-05 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.424413+0000", 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5930000000000006e-06 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.430035+0000", 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.005621643 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.466 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.434627+0000", 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0045918620000000004 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:35.435191+0000", 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00056404800000000004 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 6 2024-09-21T10:30:35.467 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:30:35.468 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:35.468 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:35 smithi032 ceph-mon[101299]: osdmap e439: 8 total, 8 up, 7 in 2024-09-21T10:30:35.468 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:35 smithi032 ceph-mon[105748]: osdmap e439: 8 total, 8 up, 7 in 2024-09-21T10:30:35.494 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:35.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:35.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:35.595 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:35.642 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:35.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:35.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:35.744 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:35.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:35.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:35.806 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:35.891 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:35.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:35.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:35.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:36.018 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:36.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:36.059 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:36.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:36.160 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:36.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 65 16 98079 98063 94.2738 48.125 0.0309644 0.0106014 2024-09-21T10:30:36.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:36.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:36 smithi045 ceph-mon[95976]: pgmap v1380: 109 pgs: 2 activating+undersized+degraded+remapped, 4 peering, 6 active+undersized+remapped, 1 remapped+peering, 2 unknown, 94 active+clean; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 28 KiB/s rd, 56 MiB/s wr, 917 op/s; 11987/288666 objects degraded (4.153%); 1.1 MiB/s, 0 objects/s recovering 2024-09-21T10:30:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:36 smithi045 ceph-mon[95976]: Health check failed: Degraded data redundancy: 11987/288666 objects degraded (4.153%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:30:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:36 smithi045 ceph-mon[95976]: osdmap e440: 8 total, 8 up, 7 in 2024-09-21T10:30:36.293 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:36 smithi032 ceph-mon[101299]: pgmap v1380: 109 pgs: 2 activating+undersized+degraded+remapped, 4 peering, 6 active+undersized+remapped, 1 remapped+peering, 2 unknown, 94 active+clean; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 28 KiB/s rd, 56 MiB/s wr, 917 op/s; 11987/288666 objects degraded (4.153%); 1.1 MiB/s, 0 objects/s recovering 2024-09-21T10:30:36.293 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:36 smithi032 ceph-mon[101299]: Health check failed: Degraded data redundancy: 11987/288666 objects degraded (4.153%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:30:36.293 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:36 smithi032 ceph-mon[101299]: osdmap e440: 8 total, 8 up, 7 in 2024-09-21T10:30:36.293 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:36 smithi032 ceph-mon[105748]: pgmap v1380: 109 pgs: 2 activating+undersized+degraded+remapped, 4 peering, 6 active+undersized+remapped, 1 remapped+peering, 2 unknown, 94 active+clean; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 28 KiB/s rd, 56 MiB/s wr, 917 op/s; 11987/288666 objects degraded (4.153%); 1.1 MiB/s, 0 objects/s recovering 2024-09-21T10:30:36.293 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:36 smithi032 ceph-mon[105748]: Health check failed: Degraded data redundancy: 11987/288666 objects degraded (4.153%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:30:36.293 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:36 smithi032 ceph-mon[105748]: osdmap e440: 8 total, 8 up, 7 in 2024-09-21T10:30:36.311 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:36.335 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:30:36.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:36.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:36.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:36.441 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:36.489 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:36.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:36.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:36.590 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:36.649 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:36.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:36.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:36.766 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:36.784 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:36.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:36.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:36.886 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:36.935 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:36.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:37.037 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:37.071 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:37.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:37.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:37.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:37.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 66 16 98780 98764 93.5088 43.8125 0.0429411 0.0106884 2024-09-21T10:30:37.207 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:37.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:37.309 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:37.340 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:37.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:37.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:37.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:37 smithi032 ceph-mon[101299]: osdmap e441: 8 total, 8 up, 7 in 2024-09-21T10:30:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:37.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:37 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:37 smithi032 ceph-mon[105748]: osdmap e441: 8 total, 8 up, 7 in 2024-09-21T10:30:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:37.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:37 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:37.474 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:37.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:37.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:37 smithi045 ceph-mon[95976]: osdmap e441: 8 total, 8 up, 7 in 2024-09-21T10:30:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:37 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:37.575 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:37.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:37.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:37.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:37.724 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:37.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:37.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:37.855 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:37.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:37.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:37.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:37.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:38.014 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:38.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:38.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:38.115 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:38.156 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:38.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:38.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:38.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 67 16 99625 99609 92.9014 52.8125 0.0111994 0.0107587 2024-09-21T10:30:38.257 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:38.302 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:38.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:38.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:38.349 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:30:38.349 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:30:38.349 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:38 smithi032 ceph-mon[105748]: pgmap v1383: 109 pgs: 2 activating+undersized+degraded+remapped, 5 peering, 5 active+undersized+remapped, 1 remapped+peering, 2 unknown, 94 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 9.5 KiB/s rd, 40 MiB/s wr, 648 op/s; 11987/290736 objects degraded (4.123%); 1.0 MiB/s, 3 objects/s recovering 2024-09-21T10:30:38.349 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:38 smithi032 ceph-mon[101299]: pgmap v1383: 109 pgs: 2 activating+undersized+degraded+remapped, 5 peering, 5 active+undersized+remapped, 1 remapped+peering, 2 unknown, 94 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 9.5 KiB/s rd, 40 MiB/s wr, 648 op/s; 11987/290736 objects degraded (4.123%); 1.0 MiB/s, 3 objects/s recovering 2024-09-21T10:30:38.370 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:30:38.370 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:30:38.393 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:30:38.393 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:30:38.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:38.415 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:30:38.416 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:30:38.438 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:30:38.438 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:30:38.442 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:38 smithi045 ceph-mon[95976]: pgmap v1383: 109 pgs: 2 activating+undersized+degraded+remapped, 5 peering, 5 active+undersized+remapped, 1 remapped+peering, 2 unknown, 94 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 9.5 KiB/s rd, 40 MiB/s wr, 648 op/s; 11987/290736 objects degraded (4.123%); 1.0 MiB/s, 3 objects/s recovering 2024-09-21T10:30:38.463 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:38.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:38.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:38.472 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:30:38.472 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:30:38.503 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:30:38.503 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:30:38.530 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:30:38.530 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:30:38.566 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:38.612 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:38.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:38.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:38.714 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:38.739 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:38.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:38.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:38.841 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:38.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:38.884 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:38.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:38.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:39.039 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:39.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:39.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:39.141 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:39.164 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:39.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:39.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 68 16 100415 100399 92.2613 49.375 0.0582945 0.0108301 2024-09-21T10:30:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:39 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3871207802' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:30:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:30:39.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:39 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3871207802' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:30:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:30:39.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:39.266 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:39.303 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:39.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:39.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:39.405 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:39.449 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:39.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:39.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:39 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3871207802' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:30:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:30:39.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:39.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:39.591 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:39.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:39.693 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:39.721 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:39.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:39.823 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:39.852 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:39.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:39.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:39.954 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:30:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:30:39] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.51.0" 2024-09-21T10:30:39.976 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:39.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:40.078 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:40.120 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:40.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:40.187 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:30:40.189 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 69 16 101223 101207 91.656 50.5 0.0329471 0.010904 2024-09-21T10:30:40.222 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:40.257 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:40.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:40.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:40.358 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:40.388 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:40.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:40.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:40 smithi032 ceph-mon[101299]: pgmap v1384: 109 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 5 active+undersized+remapped, 1 remapped+peering, 99 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 6.8 KiB/s rd, 31 MiB/s wr, 506 op/s; 11987/291453 objects degraded (4.113%); 756 KiB/s, 4 objects/s recovering 2024-09-21T10:30:40.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:40 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-09-21T10:30:40.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:40 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2972510984' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:30:40.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:40 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:30:40.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:40 smithi032 ceph-mon[105748]: pgmap v1384: 109 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 5 active+undersized+remapped, 1 remapped+peering, 99 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 6.8 KiB/s rd, 31 MiB/s wr, 506 op/s; 11987/291453 objects degraded (4.113%); 756 KiB/s, 4 objects/s recovering 2024-09-21T10:30:40.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:40 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-09-21T10:30:40.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:40 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2972510984' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:30:40.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:40 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:30:40.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:40.507 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:40 smithi045 ceph-mon[95976]: pgmap v1384: 109 pgs: 2 activating+undersized+degraded+remapped, 2 peering, 5 active+undersized+remapped, 1 remapped+peering, 99 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 6.8 KiB/s rd, 31 MiB/s wr, 506 op/s; 11987/291453 objects degraded (4.113%); 756 KiB/s, 4 objects/s recovering 2024-09-21T10:30:40.507 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:40 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-09-21T10:30:40.507 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:40 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2972510984' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:30:40.507 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:40 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:30:40.526 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:40.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:40.579 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 0, 4] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 6, 7, 5, 0] 2024-09-21T10:30:40.579 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:30:40.579 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-09-21T10:30:40.579 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd in 1 2024-09-21T10:30:40.630 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:40.669 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:40.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:40.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:40.771 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:40.802 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:40.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:40.904 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:40.963 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:40.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:40.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:41.050 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:41.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:41.123 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:41.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:41.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:41.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 70 16 102068 102052 91.1011 52.8125 0.0423908 0.01097 2024-09-21T10:30:41.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:41 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:41.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:41 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:30:41.193 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:41 smithi032 ceph-mon[105748]: osdmap e442: 8 total, 8 up, 7 in 2024-09-21T10:30:41.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:41 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:41.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:41 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:30:41.193 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:41 smithi032 ceph-mon[101299]: osdmap e442: 8 total, 8 up, 7 in 2024-09-21T10:30:41.224 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:41.251 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:41.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:41.353 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:41.404 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:41 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:41.404 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:41 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:30:41.404 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:41 smithi045 ceph-mon[95976]: osdmap e442: 8 total, 8 up, 7 in 2024-09-21T10:30:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:41.413 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:41.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:41.513 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:41.560 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:41.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:41.582 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:30:41.582 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:30:41.661 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:41.697 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:41.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:41.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:41.767 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:30:41.799 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:41.849 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:41.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:41.949 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:41.995 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:41.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:41.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:42.096 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:42.126 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:42.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:42.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:42.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 71 16 102888 102872 90.5398 51.25 0.00507976 0.011038 2024-09-21T10:30:42.227 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:42.270 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:42.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:42.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:42.355 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:30:42.371 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:42.396 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:42 smithi045 ceph-mon[95976]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 104 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 634 op/s; 11987/300084 objects degraded (3.995%); 5989/300084 objects misplaced (1.996%); 1024 KiB/s, 19 objects/s recovering 2024-09-21T10:30:42.396 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:42 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 11987/300084 objects degraded (3.995%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:30:42.398 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:42.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:42.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:42 smithi032 ceph-mon[101299]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 104 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 634 op/s; 11987/300084 objects degraded (3.995%); 5989/300084 objects misplaced (1.996%); 1024 KiB/s, 19 objects/s recovering 2024-09-21T10:30:42.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:42 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 11987/300084 objects degraded (3.995%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:30:42.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:42 smithi032 ceph-mon[105748]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 104 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 634 op/s; 11987/300084 objects degraded (3.995%); 5989/300084 objects misplaced (1.996%); 1024 KiB/s, 19 objects/s recovering 2024-09-21T10:30:42.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:42 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 11987/300084 objects degraded (3.995%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:30:42.500 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:42.542 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:42.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:42.643 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:42.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:42.688 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:42.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:42.732 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:42.789 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:42.819 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:42.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:42.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:42.920 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:42.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:42.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:42.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:42.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:30:43.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:43.080 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:43.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:43.176 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:43.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:43.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 72 16 103627 103611 89.9235 46.1875 0.0106248 0.0111127 2024-09-21T10:30:43.238 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:43.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:43.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:43.341 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:43.366 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:43.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:43.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:43.468 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:43.490 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:43.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:43.591 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:43.616 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:43 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:30:43.387+0000 7f532c355640 -1 mon.a@0(leader).osd e442 definitely_dead 0 2024-09-21T10:30:43.617 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:43.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:43.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:43.719 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:43.743 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:43.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:43.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:43.844 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:43.874 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:43.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:43.975 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:43.995 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:44.018 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:44.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:44.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:44.120 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:44.150 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:44.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:44.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:44.190 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 73 16 104127 104111 89.1197 31.25 0.0173951 0.0112138 2024-09-21T10:30:44.197 INFO:teuthology.orchestra.run.smithi032.stderr:marked in osd.1. 2024-09-21T10:30:44.252 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:44.278 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:44.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:44.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:44.380 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:44.403 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:44.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:44.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:44 smithi032 ceph-mon[101299]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 37 MiB/s wr, 592 op/s; 11987/303246 objects degraded (3.953%); 5989/303246 objects misplaced (1.975%); 811 KiB/s, 15 objects/s recovering 2024-09-21T10:30:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:44 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1104731686' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:30:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:44 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:30:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:44 smithi032 ceph-mon[105748]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 37 MiB/s wr, 592 op/s; 11987/303246 objects degraded (3.953%); 5989/303246 objects misplaced (1.975%); 811 KiB/s, 15 objects/s recovering 2024-09-21T10:30:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:44 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1104731686' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:30:44.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:44 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:30:44.505 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:44.527 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:44.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:44.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:44.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:44 smithi045 ceph-mon[95976]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 37 MiB/s wr, 592 op/s; 11987/303246 objects degraded (3.953%); 5989/303246 objects misplaced (1.975%); 811 KiB/s, 15 objects/s recovering 2024-09-21T10:30:44.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:44 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1104731686' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:30:44.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:44 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:30:44.628 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:44.650 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:44.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:44.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:44.752 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:44.776 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:44.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:44.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:44.877 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:44.901 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:44.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:44.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:45.003 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:45.027 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:45.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:45.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:45.129 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:45.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:45.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:45.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:45.171 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:30:45.171 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:30:45.193 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 74 16 104164 104148 87.9431 2.3125 0.00386854 0.0112133 2024-09-21T10:30:45.216 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:30:45.216 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:30:45.245 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:30:45.246 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:30:45.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:45.267 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:30:45.267 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:30:45.283 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:45.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:45.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:45.288 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:30:45.288 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:30:45.316 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:30:45.316 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:30:45.336 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:45 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:30:45.336 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:45 smithi032 ceph-mon[101299]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T10:30:45.336 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:45 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2912522274' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:45.336 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:45 smithi032 ceph-mon[101299]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T10:30:45.336 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:45 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:30:45.336 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:45 smithi032 ceph-mon[105748]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T10:30:45.336 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:45 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2912522274' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:45.336 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:45 smithi032 ceph-mon[105748]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T10:30:45.338 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:45 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:30:45.338 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:45 smithi045 ceph-mon[95976]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T10:30:45.338 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:45 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2912522274' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:45.338 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:45 smithi045 ceph-mon[95976]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T10:30:45.345 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:30:45.345 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:30:45.378 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:30:45.378 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:30:45.385 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:45.423 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:45.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:45.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:45.525 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:45.567 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:45.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:45.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:45.669 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:45.710 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:45.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:45.811 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:45.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:45.857 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:45.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:45.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:46.001 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:46.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:46.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:46.102 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:46.140 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:46.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:46.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 75 16 105146 105130 87.5888 61.375 0.0160554 0.011412 2024-09-21T10:30:46.241 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:46 smithi045 ceph-mon[95976]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 100 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s; 5870/310125 objects misplaced (1.893%); 1.7 MiB/s, 28 objects/s recovering 2024-09-21T10:30:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:46 smithi045 ceph-mon[95976]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11987/303246 objects degraded (3.953%), 2 pgs degraded) 2024-09-21T10:30:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:46 smithi045 ceph-mon[95976]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T10:30:46.289 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:46.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:46.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:46.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:46.429 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:46.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:46.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:46.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:46 smithi032 ceph-mon[105748]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 100 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s; 5870/310125 objects misplaced (1.893%); 1.7 MiB/s, 28 objects/s recovering 2024-09-21T10:30:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:46 smithi032 ceph-mon[105748]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11987/303246 objects degraded (3.953%), 2 pgs degraded) 2024-09-21T10:30:46.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:46 smithi032 ceph-mon[105748]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T10:30:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:46 smithi032 ceph-mon[101299]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 100 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s; 5870/310125 objects misplaced (1.893%); 1.7 MiB/s, 28 objects/s recovering 2024-09-21T10:30:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:46 smithi032 ceph-mon[101299]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11987/303246 objects degraded (3.953%), 2 pgs degraded) 2024-09-21T10:30:46.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:46 smithi032 ceph-mon[101299]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T10:30:46.530 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:46.568 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:46.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:46.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:46.574 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:30:46.574 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:30:46.574 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:30:46.574 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:30:46.574 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.574 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:126173 8.d 8:bbe71d92:::benchmark_data_smithi032_233595_object26328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:21:52.546485+0000", 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 534.02269339600002, 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.416189632, 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 126173 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546485+0000", 2024-09-21T10:30:46.575 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546485+0000", 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546483+0000", 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.576 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546489+0000", 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4659999999999996e-06 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546501+0000", 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1599e-05 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.546508+0000", 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1539999999999996e-06 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.599690+0000", 2024-09-21T10:30:46.577 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.053182116000000001 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:52.601485+0000", 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0017949839999999999 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.233600+0000", 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.63211453500000003 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.578 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.236756+0000", 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031565199999999999 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.933863+0000", 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.69710698299999996 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.936865+0000", 2024-09-21T10:30:46.579 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0030012799999999998 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.942213+0000", 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0053482879999999997 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.953349+0000", 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011136102 2024-09-21T10:30:46.580 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.953630+0000", 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000281 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.954544+0000", 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00091381800000000005 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.955453+0000", 2024-09-21T10:30:46.581 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00090889200000000001 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.956882+0000", 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0014294450000000001 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:53.962674+0000", 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0057920460000000003 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.582 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:127125 8.d 8:b41a31bb:::benchmark_data_smithi032_233595_object27280:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:21:56.132642+0000", 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 530.43653552900003, 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3750970520000001, 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 127125 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.583 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132642+0000", 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132642+0000", 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132640+0000", 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.584 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132649+0000", 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2989999999999997e-06 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132665+0000", 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6138999999999999e-05 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.132673+0000", 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9570000000000002e-06 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.585 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.190478+0000", 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.057805095000000001 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.191875+0000", 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001396867 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.759185+0000", 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.56731008699999996 2024-09-21T10:30:46.586 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:56.762972+0000", 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0037875370000000001 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.466882+0000", 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.70391016799999995 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.469411+0000", 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0025282450000000001 2024-09-21T10:30:46.587 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.588 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.588 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.474173+0000", 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0047627809999999998 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.498463+0000", 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.024289439999999999 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.499344+0000", 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00088115700000000003 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.589 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.500162+0000", 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00081813700000000003 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.500964+0000", 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00080155899999999997 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.502290+0000", 2024-09-21T10:30:46.590 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001326144 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:21:57.507739+0000", 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0054495749999999999 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "MOSDPGPush(8.d 348/339 [PushOp(8:bb1880f6:::benchmark_data_smithi032_233595_object29338:head, version: 314'1888, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:bb1880f6:::benchmark_data_smithi032_233595_object29338:head@314'1888, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:02.376911+0000", 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 524.19226655900002, 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5228334210000001, 2024-09-21T10:30:46.591 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376911+0000", 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376911+0000", 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.592 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376911+0000", 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376950+0000", 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8979000000000001e-05 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376964+0000", 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4409e-05 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:02.376977+0000", 2024-09-21T10:30:46.593 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.312e-05 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.889091+0000", 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.51211345 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.889396+0000", 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00030527700000000001 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.899745+0000", 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010348697 2024-09-21T10:30:46.594 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:129183 8.d 8:bb1880f6:::benchmark_data_smithi032_233595_object29338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:03.381897+0000", 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 523.18728088600005, 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2215354430000001, 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 129183 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.595 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381897+0000", 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381897+0000", 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381895+0000", 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.596 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381903+0000", 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.165e-06 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381917+0000", 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4945e-05 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.381925+0000", 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5510000000000002e-06 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.597 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.598817+0000", 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.21689214900000001 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.600951+0000", 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002134141 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.900025+0000", 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.29907400099999998 2024-09-21T10:30:46.598 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:03.902880+0000", 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0028542099999999998 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.571519+0000", 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.66863905599999995 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.573803+0000", 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022840120000000002 2024-09-21T10:30:46.599 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.578058+0000", 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0042553729999999998 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.593724+0000", 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015666253000000002 2024-09-21T10:30:46.600 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.601 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.601 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.601 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.593986+0000", 2024-09-21T10:30:46.601 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00026158399999999998 2024-09-21T10:30:46.601 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.594907+0000", 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00092106000000000002 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.595766+0000", 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00085888999999999996 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.602 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.597240+0000", 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0014741159999999999 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.603432+0000", 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0061925740000000002 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:129369 8.d 8:b32a093f:::benchmark_data_smithi032_233595_object29524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:04.040016+0000", 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 522.52916163700002, 2024-09-21T10:30:46.603 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.187289474, 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 129369 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040016+0000", 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.604 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040016+0000", 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040015+0000", 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040022+0000", 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7050000000000002e-06 2024-09-21T10:30:46.605 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040034+0000", 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2129e-05 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.040042+0000", 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0779999999999996e-06 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.072059+0000", 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.032017589999999999 2024-09-21T10:30:46.606 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.073424+0000", 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001364765 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.524208+0000", 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.45078337699999999 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:04.527081+0000", 2024-09-21T10:30:46.607 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0028734640000000001 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.181266+0000", 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.65418491099999998 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.184177+0000", 2024-09-21T10:30:46.608 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029113770000000001 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.189517+0000", 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0053394569999999997 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.216225+0000", 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.026708104999999999 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.217266+0000", 2024-09-21T10:30:46.609 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001040736 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.218049+0000", 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00078369599999999996 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.219162+0000", 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0011127120000000001 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.220543+0000", 2024-09-21T10:30:46.610 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0013809250000000001 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:05.227306+0000", 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0067628059999999997 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:130475 8.d 8:b0e43261:::benchmark_data_smithi032_233595_object30630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:07.833409+0000", 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 518.73576908500002, 2024-09-21T10:30:46.611 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2543778620000001, 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 130475 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833409+0000", 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.612 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833409+0000", 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833407+0000", 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833414+0000", 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3419999999999996e-06 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.613 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833426+0000", 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1956999999999999e-05 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.833431+0000", 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6250000000000004e-06 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.844081+0000", 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010650049999999999 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:07.845500+0000", 2024-09-21T10:30:46.614 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001418348 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:08.478678+0000", 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.63317818199999998 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:08.482182+0000", 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0035043309999999999 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.038948+0000", 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.55676618899999997 2024-09-21T10:30:46.615 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.041858+0000", 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029101019999999999 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.049048+0000", 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0071896709999999999 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.077242+0000", 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.028194252 2024-09-21T10:30:46.616 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.078305+0000", 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0010624269999999999 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.079245+0000", 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00093980400000000003 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.080257+0000", 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0010122460000000001 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.617 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.081835+0000", 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001577651 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:09.087787+0000", 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0059521790000000002 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.618 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:137893 8.d 8:bc2e2b1d:::benchmark_data_smithi032_233595_object38048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:28.962493+0000", 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 497.60668454799998, 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1154502959999999, 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 137893 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.619 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962493+0000", 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962493+0000", 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962492+0000", 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.620 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962498+0000", 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.891e-06 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962509+0000", 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1324e-05 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.962515+0000", 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9710000000000001e-06 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.621 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.986383+0000", 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.023867616000000001 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:28.987610+0000", 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001227204 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:29.484804+0000", 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.49719403600000001 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.622 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:29.487315+0000", 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002511685 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.028673+0000", 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.54135758599999995 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.031518+0000", 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0028447749999999999 2024-09-21T10:30:46.623 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.036720+0000", 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.005202351 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.064312+0000", 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.027591844000000001 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.065138+0000", 2024-09-21T10:30:46.624 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00082640299999999997 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.065916+0000", 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00077757800000000001 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.068661+0000", 2024-09-21T10:30:46.625 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002744771 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.069887+0000", 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0012257920000000001 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.077944+0000", 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0080571140000000006 2024-09-21T10:30:46.626 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:138528 8.d 8:b42d3168:::benchmark_data_smithi032_233595_object38683:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:30.651318+0000", 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 495.91785992699999, 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.17798106, 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.627 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 138528 2024-09-21T10:30:46.628 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.628 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.628 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.628 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.628 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651318+0000", 2024-09-21T10:30:46.628 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.628 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.628 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.628 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651318+0000", 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651316+0000", 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.629 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651323+0000", 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9249999999999998e-06 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651334+0000", 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0733999999999999e-05 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.651342+0000", 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1759999999999993e-06 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.630 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.693935+0000", 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.042592334000000003 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:30.695200+0000", 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001265555 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.226631+0000", 2024-09-21T10:30:46.631 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.53143101500000001 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.229128+0000", 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002496322 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.807497+0000", 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.57836990700000002 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.809660+0000", 2024-09-21T10:30:46.632 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0021629309999999999 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.813036+0000", 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0033755149999999999 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.821737+0000", 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087014469999999993 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.633 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.822419+0000", 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00068219000000000005 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.822626+0000", 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00020688099999999999 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.823351+0000", 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00072413800000000004 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.634 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.824503+0000", 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001152246 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:31.829299+0000", 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.004796285 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:139301 8.d 8:bd7927c9:::benchmark_data_smithi032_233595_object39456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-21T10:30:46.635 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:22:32.637011+0000", 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 493.932166666, 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99859100099999998, 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 139301 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637011+0000", 2024-09-21T10:30:46.636 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.637 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.637 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.637 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.637 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637011+0000", 2024-09-21T10:30:46.637 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.637 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.637 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.637 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637010+0000", 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637016+0000", 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6289999999999998e-06 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637026+0000", 2024-09-21T10:30:46.638 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.094e-05 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.637031+0000", 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8729999999999998e-06 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.652132+0000", 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.015100891 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:30:46.639 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:32.653357+0000", 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001224356 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.105349+0000", 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.45199279599999997 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for rw locks", 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.107505+0000", 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0021551140000000001 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.640 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.605380+0000", 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.49787499699999999 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.607090+0000", 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001710575 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,6,7", 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.610073+0000", 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029829330000000001 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.641 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.629265+0000", 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.019192331999999999 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.629458+0000", 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000192497 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.642 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.630105+0000", 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000647633 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.630687+0000", 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00058175499999999999 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.631651+0000", 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00096404699999999999 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.643 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:22:33.635602+0000", 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0039509749999999998 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174180 8.8 8:1cb2aae1:::benchmark_data_smithi032_233595_object74335:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666281+0000", 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 439.902897376, 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0395073349999999, 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.644 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174180 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666281+0000", 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666281+0000", 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.645 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666279+0000", 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666288+0000", 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4409999999999993e-06 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.646 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666297+0000", 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.9350000000000006e-06 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666304+0000", 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3589999999999996e-06 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676089+0000", 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0097852890000000008 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.647 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.648 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.648 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676099+0000", 2024-09-21T10:30:46.648 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4630000000000007e-06 2024-09-21T10:30:46.648 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.648 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.648 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.648 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.705704+0000", 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.029605076 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.705788+0000", 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4192999999999998e-05 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174191 8.8 8:1067ce08:::benchmark_data_smithi032_233595_object74346:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666311+0000", 2024-09-21T10:30:46.649 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 439.90286641799997, 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.039912868, 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174191 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666311+0000", 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.650 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666311+0000", 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666311+0000", 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666313+0000", 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2850000000000001e-06 2024-09-21T10:30:46.651 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666319+0000", 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3929999999999999e-06 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666323+0000", 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4279999999999998e-06 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676281+0000", 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0099576779999999993 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.652 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676291+0000", 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6099999999999995e-06 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706038+0000", 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0297476919999999 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706224+0000", 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000186152 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.653 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174200 8.8 8:1e9d63d6:::benchmark_data_smithi032_233595_object74355:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666327+0000", 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 439.90285067299999, 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0398621889999999, 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174200 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.654 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666327+0000", 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666327+0000", 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666327+0000", 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.655 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666329+0000", 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9589999999999998e-06 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666334+0000", 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1170000000000003e-06 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666338+0000", 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.391e-06 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.656 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.657 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.657 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676317+0000", 2024-09-21T10:30:46.657 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0099783319999999995 2024-09-21T10:30:46.657 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.657 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.657 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.657 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676323+0000", 2024-09-21T10:30:46.657 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7009999999999999e-06 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706125+0000", 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.029801084 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706189+0000", 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4919000000000006e-05 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.658 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174202 8.8 8:1d58c5d6:::benchmark_data_smithi032_233595_object74357:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666343+0000", 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 439.902835396, 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.039984061, 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174202 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.659 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666343+0000", 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666343+0000", 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666342+0000", 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:46.660 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666344+0000", 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8360000000000001e-06 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666349+0000", 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9470000000000002e-06 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666353+0000", 2024-09-21T10:30:46.661 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0729999999999998e-06 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676345+0000", 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0099917489999999994 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676351+0000", 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5019999999999998e-06 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.662 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706272+0000", 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.029920929 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706327+0000", 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4351999999999997e-05 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174205 8.8 8:1099f92f:::benchmark_data_smithi032_233595_object74360:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.666357+0000", 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 439.90282124100003, 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0550764530000001, 2024-09-21T10:30:46.663 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174205 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666357+0000", 2024-09-21T10:30:46.664 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666357+0000", 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666356+0000", 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.665 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666358+0000", 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8560000000000001e-06 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666363+0000", 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.096e-06 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.666367+0000", 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0539999999999996e-06 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.666 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676801+0000", 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.01043375 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.676830+0000", 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8911000000000002e-05 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706354+0000", 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.029523974 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for missing object", 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706455+0000", 2024-09-21T10:30:46.667 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010135499999999999 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721358+0000", 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014903035 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721433+0000", 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4714000000000003e-05 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174244 8.8 8:1bf2b167:::benchmark_data_smithi032_233595_object74399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:30:46.668 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.722448+0000", 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 439.84672960199998, 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99908222599999996, 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174244 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722448+0000", 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.669 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722448+0000", 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722447+0000", 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.670 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722453+0000", 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6800000000000004e-06 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722469+0000", 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5333e-05 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.722476+0000", 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9970000000000001e-06 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.742937+0000", 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.020461098 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.742946+0000", 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2920000000000001e-06 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706508+0000", 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96356169400000002 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.672 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706631+0000", 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012306 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.706824+0000", 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000193325 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.717553+0000", 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010728946 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.673 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.717566+0000", 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2446000000000001e-05 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721454+0000", 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0038887230000000002 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721483+0000", 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8660999999999999e-05 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.674 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721531+0000", 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7509000000000002e-05 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:174257 8.8 8:1e3e855f:::benchmark_data_smithi032_233595_object74412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:26.737607+0000", 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 439.83157130000001, 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99780132499999996, 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.675 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 174257 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737607+0000", 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737607+0000", 2024-09-21T10:30:46.676 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737605+0000", 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737617+0000", 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2109e-05 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737629+0000", 2024-09-21T10:30:46.677 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2557e-05 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.737636+0000", 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7490000000000001e-06 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.747542+0000", 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0099056779999999994 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:26.747552+0000", 2024-09-21T10:30:46.678 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0183000000000001e-05 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.713711+0000", 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.96615876499999997 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.713848+0000", 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013773400000000001 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.714001+0000", 2024-09-21T10:30:46.679 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000152538 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.721431+0000", 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0074296529999999996 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.728270+0000", 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0068392119999999999 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.735248+0000", 2024-09-21T10:30:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0069787089999999996 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.735306+0000", 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7936000000000002e-05 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:27.735408+0000", 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000101568 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.681 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:191427 8.d 8:b66cbf8c:::benchmark_data_smithi032_233595_object91582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:47.766812+0000", 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 418.802366253, 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.050450109, 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 191427 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766812+0000", 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.682 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766812+0000", 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766810+0000", 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.683 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.683 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:46.684 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766818+0000", 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5530000000000004e-06 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766834+0000", 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5931e-05 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.766841+0000", 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0180000000000004e-06 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.685 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.774617+0000", 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0077766019999999996 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.774704+0000", 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6446999999999994e-05 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.805821+0000", 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.031117743 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.686 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.805945+0000", 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012376499999999999 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.806069+0000", 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012341899999999999 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.813666+0000", 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0075971850000000002 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.687 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817135+0000", 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0034688340000000001 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817206+0000", 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1890000000000005e-05 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817238+0000", 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1825000000000001e-05 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.688 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817262+0000", 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3470999999999999e-05 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:191444 8.d 8:b32f16b0:::benchmark_data_smithi032_233595_object91599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:47.787356+0000", 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 418.78182221700001, 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0395182649999999, 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.689 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 191444 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787356+0000", 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787356+0000", 2024-09-21T10:30:46.690 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787354+0000", 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787361+0000", 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7190000000000004e-06 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787379+0000", 2024-09-21T10:30:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8063e-05 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787388+0000", 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5990000000000002e-06 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787442+0000", 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4570000000000001e-05 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.692 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.787452+0000", 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8090000000000004e-06 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.810170+0000", 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0227181999999999 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.810272+0000", 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000102013 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:30:46.693 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.810342+0000", 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0140000000000003e-05 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817302+0000", 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0069597390000000004 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817312+0000", 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4050000000000006e-06 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.694 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826810+0000", 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094981749999999993 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826840+0000", 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0633000000000003e-05 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826874+0000", 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3541000000000003e-05 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.695 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:191477 8.d 8:b5a49fb0:::benchmark_data_smithi032_233595_object91632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:47.820759+0000", 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 418.74841876400001, 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.006215826, 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 191477 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.696 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820759+0000", 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820759+0000", 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820757+0000", 2024-09-21T10:30:46.697 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820765+0000", 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6109999999999996e-06 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820777+0000", 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2283000000000001e-05 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.698 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820785+0000", 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1690000000000003e-06 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820832+0000", 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7435000000000002e-05 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.820841+0000", 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.1039999999999996e-06 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.699 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.813711+0000", 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99286991099999999 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.813806+0000", 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4640000000000002e-05 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.813884+0000", 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8230000000000001e-05 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.700 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817319+0000", 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0034351300000000002 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817350+0000", 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0809999999999998e-05 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826919+0000", 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0095689750000000004 2024-09-21T10:30:46.701 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826950+0000", 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1075999999999999e-05 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.826975+0000", 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.5123999999999999e-05 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.702 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.35586.0:191490 8.d 8:b4e2352a:::benchmark_data_smithi032_233595_object91645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:23:47.837183+0000", 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 418.73199466699998, 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98991871600000003, 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.35586", 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1479172042", 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 191490 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:30:46.703 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837183+0000", 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837183+0000", 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837182+0000", 2024-09-21T10:30:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837188+0000", 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.874e-06 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837203+0000", 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5675000000000002e-05 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837212+0000", 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.7110000000000001e-06 2024-09-21T10:30:46.705 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837265+0000", 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3266000000000001e-05 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:47.837274+0000", 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4370000000000007e-06 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817384+0000", 2024-09-21T10:30:46.706 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98011053000000004 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817458+0000", 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3233999999999997e-05 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 2,7", 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.817568+0000", 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000109917 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827010+0000", 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0094422699999999991 2024-09-21T10:30:46.707 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827015+0000", 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2279999999999998e-06 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827047+0000", 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1959999999999999e-05 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827082+0000", 2024-09-21T10:30:46.708 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5169000000000002e-05 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:23:48.827102+0000", 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9758e-05 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:30:46.709 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:30:46.715 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-09-21T10:30:46.719 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:46.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:46.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:46.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:46.866 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:46.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:46.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:46.968 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:47.015 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:47.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:47.117 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:47.155 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:47.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:47.193 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 76 16 106104 106088 87.2242 59.875 0.00956056 0.0114596 2024-09-21T10:30:47.257 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:47.298 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:47.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:47.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:47.431 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:47.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:47.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:47.532 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:47.567 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:47.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:47.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:47.669 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:47.709 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:30:47.726 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:47.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:47.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:47.828 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:47.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:47.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:47.987 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:48.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:48.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:48.122 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:48.154 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:48.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:48.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 77 16 107108 107092 86.9061 62.75 0.0091069 0.011501 2024-09-21T10:30:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:47 smithi032 ceph-mon[101299]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+remapped, 4 remapped+peering, 13 peering, 91 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s; 5870/312438 objects misplaced (1.879%); 1.2 MiB/s, 19 objects/s recovering 2024-09-21T10:30:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:47 smithi032 ceph-mon[101299]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T10:30:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:47 smithi032 ceph-mon[105748]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+remapped, 4 remapped+peering, 13 peering, 91 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s; 5870/312438 objects misplaced (1.879%); 1.2 MiB/s, 19 objects/s recovering 2024-09-21T10:30:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:47 smithi032 ceph-mon[105748]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T10:30:48.255 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:48.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:47 smithi045 ceph-mon[95976]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+remapped, 4 remapped+peering, 13 peering, 91 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s; 5870/312438 objects misplaced (1.879%); 1.2 MiB/s, 19 objects/s recovering 2024-09-21T10:30:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:47 smithi045 ceph-mon[95976]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T10:30:48.301 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:48.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:48.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:48.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:48.435 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:48.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:48.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:48.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:48.562 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:48.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:48.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:48.708 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:48.752 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:48.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:48.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:48.853 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:48.894 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:48.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:48.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:48.996 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:49.039 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:49.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:49.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:49.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:49.180 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:49.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:49.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:49.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 78 16 108128 108112 86.6092 63.75 0.0127986 0.0115415 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[101299]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/696623895' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[105748]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/696623895' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:49.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:48 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:49.282 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:48 smithi045 ceph-mon[95976]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T10:30:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:30:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:30:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:48 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/696623895' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:48 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:30:49.317 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:49.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:49.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:49.419 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:49.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:49.458 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:49.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:49.557 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:30:49.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:49.599 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:49.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:49.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:49.695 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:30:49] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-09-21T10:30:49.701 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:49.737 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:49.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:49.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:30:49.839 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:49.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:49.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:49.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:50.006 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:50.052 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:50.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:50.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:50.154 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:50.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:50.183 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:50.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:50.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 79 16 109136 109120 86.3104 63 0.0260735 0.011581 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[105748]: pgmap v1395: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 remapped+peering, 13 peering, 90 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 18 MiB/s wr, 292 op/s; 10/313638 objects degraded (0.003%); 5870/313638 objects misplaced (1.872%); 0 B/s, 0 objects/s recovering 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[105748]: Health check failed: Degraded data redundancy: 10/313638 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[105748]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[101299]: pgmap v1395: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 remapped+peering, 13 peering, 90 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 18 MiB/s wr, 292 op/s; 10/313638 objects degraded (0.003%); 5870/313638 objects misplaced (1.872%); 0 B/s, 0 objects/s recovering 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[101299]: Health check failed: Degraded data redundancy: 10/313638 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:30:50.253 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:49 smithi032 ceph-mon[101299]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T10:30:50.284 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:49 smithi045 ceph-mon[95976]: pgmap v1395: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 remapped+peering, 13 peering, 90 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 18 MiB/s wr, 292 op/s; 10/313638 objects degraded (0.003%); 5870/313638 objects misplaced (1.872%); 0 B/s, 0 objects/s recovering 2024-09-21T10:30:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:49 smithi045 ceph-mon[95976]: Health check failed: Degraded data redundancy: 10/313638 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:30:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:49 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:30:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:49 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:30:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:49 smithi045 ceph-mon[95976]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T10:30:50.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:50.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:50.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:50.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:50.474 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:50.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:50.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:50.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:50.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:50.610 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:50.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:50.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:50.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:50.749 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:50.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:50.850 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:50.888 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:50.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:50.990 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:51.025 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:51.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:51.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:51.056 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:51.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:51.158 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:51.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:51.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:51.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:2024-09-21T10:30:51.193287+0000 min lat: 0.00102285 max lat: 2.23837 avg lat: 0.0116188 2024-09-21T10:30:51.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:30:51.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 80 16 110104 110088 85.9877 60.5 0.009661 0.0116188 2024-09-21T10:30:51.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:51.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:51.299 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:51.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:51.399 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:51.424 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:51.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:51.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:51.526 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:51.548 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:51.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:51.650 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:51.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:51.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:51.716 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 0, 4, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 6, 7, 5, 0] 2024-09-21T10:30:51.716 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:30:51.716 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-21T10:30:51.716 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:30:51.716 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:30:51.716 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-21T10:30:51.775 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:51.797 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:51.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:51.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:51.899 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:51.926 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:51.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:51.954 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:30:52.027 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:52.034 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:51 smithi045 ceph-mon[95976]: pgmap v1397: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 101 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 987/323019 objects degraded (0.306%); 154/323019 objects misplaced (0.048%); 4.3 MiB/s, 66 objects/s recovering 2024-09-21T10:30:52.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:51 smithi045 ceph-mon[95976]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T10:30:52.046 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:30:52.047 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:30:52.047 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 0 2024-09-21T10:30:52.047 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:30:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:52.066 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:52.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:52.167 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:52.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 81 16 110432 110416 85.1793 20.5 0.0103089 0.0117344 2024-09-21T10:30:52.197 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:52.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:52.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:52.299 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:52.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:52.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:52.361 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:51 smithi032 ceph-mon[101299]: pgmap v1397: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 101 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 987/323019 objects degraded (0.306%); 154/323019 objects misplaced (0.048%); 4.3 MiB/s, 66 objects/s recovering 2024-09-21T10:30:52.361 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:51 smithi032 ceph-mon[101299]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T10:30:52.361 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:51 smithi032 ceph-mon[105748]: pgmap v1397: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 101 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 987/323019 objects degraded (0.306%); 154/323019 objects misplaced (0.048%); 4.3 MiB/s, 66 objects/s recovering 2024-09-21T10:30:52.361 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:51 smithi032 ceph-mon[105748]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T10:30:52.437 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:52.477 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:52.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:52.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:52.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:52.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:52.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:52.723 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:52.733 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:30:52.765 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:52.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:52.779 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:30:52.867 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:52.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:52.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:52.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:52.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:53.035 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:53.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:53.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:53.136 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:53.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:53.183 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:53.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:53.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 82 16 111431 111415 84.9019 62.4375 0.00770503 0.0117738 2024-09-21T10:30:53.284 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:52 smithi045 ceph-mon[95976]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T10:30:53.316 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:53.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:53.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:53.381 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:53.417 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:53.454 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:52 smithi032 ceph-mon[105748]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T10:30:53.455 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:52 smithi032 ceph-mon[101299]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T10:30:53.466 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:53.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:53.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:53.569 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:53.607 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:53.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:53.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:53.740 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:53.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:53.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:53.842 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:53.875 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:53.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:53.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:53.977 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:54.019 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:54.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:54.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:54.121 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:54.160 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:54.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:54.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:54.194 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 83 16 112320 112304 84.5484 55.5625 0.027121 0.0118218 2024-09-21T10:30:54.262 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:54.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:53 smithi045 ceph-mon[95976]: pgmap v1400: 109 pgs: 4 peering, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 5.2 KiB/s rd, 57 MiB/s wr, 918 op/s; 977/326760 objects degraded (0.299%); 6.0 MiB/s, 99 objects/s recovering 2024-09-21T10:30:54.297 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:53 smithi032 ceph-mon[101299]: pgmap v1400: 109 pgs: 4 peering, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 5.2 KiB/s rd, 57 MiB/s wr, 918 op/s; 977/326760 objects degraded (0.299%); 6.0 MiB/s, 99 objects/s recovering 2024-09-21T10:30:54.298 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:53 smithi032 ceph-mon[105748]: pgmap v1400: 109 pgs: 4 peering, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 5.2 KiB/s rd, 57 MiB/s wr, 918 op/s; 977/326760 objects degraded (0.299%); 6.0 MiB/s, 99 objects/s recovering 2024-09-21T10:30:54.298 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:30:54.298 INFO:teuthology.orchestra.run.smithi045.stdout: "success": "" 2024-09-21T10:30:54.298 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:30:54.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:54.303 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:54.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:54.303 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:30:54.303 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:30:54.356 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:30:54.356 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:30:54.399 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:30:54.399 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:30:54.403 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:54.432 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:30:54.432 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:30:54.439 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:54.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:54.485 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:30:54.485 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:30:54.522 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:30:54.522 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:30:54.541 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:54.560 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:30:54.561 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:30:54.576 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:54.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:54.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:54.595 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:30:54.595 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:30:54.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:54.716 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:54.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:54.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:54.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:54.933 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:54.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:55.034 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:55.070 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:55.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:55.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:55.172 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:30:55.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 84 16 113212 113196 84.2049 55.75 0.00542918 0.0118701 2024-09-21T10:30:55.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:55.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:30:55.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:54 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4071094056' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:55.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:55.347 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:55.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:55.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:55.449 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:55.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:54 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4071094056' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:55.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:54 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4071094056' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:30:55.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:55.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:55.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:55.583 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:55.613 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:55.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:55.714 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:55.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:55.761 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:55.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:55.862 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:55.903 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:55.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:55.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:56.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:56.046 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:56.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:56.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:56.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:56.175 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:56.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:56.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:56.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 85 16 114144 114128 83.8996 58.25 0.0043695 0.0119012 2024-09-21T10:30:56.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:56 smithi032 ceph-mon[105748]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 4 peering, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.2 KiB/s rd, 70 MiB/s wr, 1.12k op/s; 490/333723 objects degraded (0.147%); 75/333723 objects misplaced (0.022%); 9.0 MiB/s, 148 objects/s recovering 2024-09-21T10:30:56.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:56 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 490/333723 objects degraded (0.147%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:30:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:56 smithi032 ceph-mon[101299]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 4 peering, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.2 KiB/s rd, 70 MiB/s wr, 1.12k op/s; 490/333723 objects degraded (0.147%); 75/333723 objects misplaced (0.022%); 9.0 MiB/s, 148 objects/s recovering 2024-09-21T10:30:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:56 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 490/333723 objects degraded (0.147%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:30:56.277 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:56 smithi045 ceph-mon[95976]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 4 peering, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.2 KiB/s rd, 70 MiB/s wr, 1.12k op/s; 490/333723 objects degraded (0.147%); 75/333723 objects misplaced (0.022%); 9.0 MiB/s, 148 objects/s recovering 2024-09-21T10:30:56.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:56 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 490/333723 objects degraded (0.147%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:30:56.300 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:56.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:56.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:56.402 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:56.424 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:56.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:56.526 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:56.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:56.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:56.646 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:56.651 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:56.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:56.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:56.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:56.801 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:56.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:56.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:56.903 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:56.925 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:56.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:57.027 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:57.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:57.063 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:57.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:57.163 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:57.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 86 16 114304 114288 83.0404 10 0.0048861 0.0120372 2024-09-21T10:30:57.196 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:57.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:57.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:57 smithi032 ceph-mon[101299]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T10:30:57.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:57 smithi032 ceph-mon[105748]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T10:30:57.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:57 smithi045 ceph-mon[95976]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T10:30:57.298 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:57.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:57.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:57.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:57.472 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:57.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:57.573 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:57.609 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:57.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:57.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:57.745 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:57.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:57.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:57.847 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:57.889 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:57.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:57.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:57.991 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:58.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:58.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:58.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:58.138 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:58.173 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:58.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:58.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:58.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 87 16 115346 115330 82.8344 65.125 0.0120504 0.0120675 2024-09-21T10:30:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:58 smithi032 ceph-mon[105748]: pgmap v1403: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.2 KiB/s rd, 51 MiB/s wr, 820 op/s; 490/337746 objects degraded (0.145%); 75/337746 objects misplaced (0.022%); 5.8 MiB/s, 97 objects/s recovering 2024-09-21T10:30:58.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:58 smithi032 ceph-mon[105748]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T10:30:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:58 smithi032 ceph-mon[101299]: pgmap v1403: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.2 KiB/s rd, 51 MiB/s wr, 820 op/s; 490/337746 objects degraded (0.145%); 75/337746 objects misplaced (0.022%); 5.8 MiB/s, 97 objects/s recovering 2024-09-21T10:30:58.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:58 smithi032 ceph-mon[101299]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T10:30:58.274 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:58.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:58 smithi045 ceph-mon[95976]: pgmap v1403: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.2 KiB/s rd, 51 MiB/s wr, 820 op/s; 490/337746 objects degraded (0.145%); 75/337746 objects misplaced (0.022%); 5.8 MiB/s, 97 objects/s recovering 2024-09-21T10:30:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:58 smithi045 ceph-mon[95976]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T10:30:58.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:58.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:58.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:58.414 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:58.455 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:58.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:58.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:58.558 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:30:58.609 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:58.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:58.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:30:58.666 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:30:58.711 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:30:58.750 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:58.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:30:58.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:58.886 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:58.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:58.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:58.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:30:59.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:59.019 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:59.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:30:59.120 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:30:59.152 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:59.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:59.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:30:59.162 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:59 smithi032 ceph-mon[105748]: 1.10 scrub starts 2024-09-21T10:30:59.163 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:30:59 smithi032 ceph-mon[105748]: 1.10 scrub ok 2024-09-21T10:30:59.163 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:59 smithi032 ceph-mon[101299]: 1.10 scrub starts 2024-09-21T10:30:59.163 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:30:59 smithi032 ceph-mon[101299]: 1.10 scrub ok 2024-09-21T10:30:59.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 88 16 116316 116300 82.582 60.625 0.00360325 0.0120917 2024-09-21T10:30:59.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:59.276 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:59.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:59.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:59.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:59 smithi045 ceph-mon[95976]: 1.10 scrub starts 2024-09-21T10:30:59.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:30:59 smithi045 ceph-mon[95976]: 1.10 scrub ok 2024-09-21T10:30:59.378 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:59.383 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:30:59.383 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:30:59.400 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:59.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:59.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:59.501 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:59.523 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:59.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:59.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:59.625 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:30:59.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:59.647 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:59.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:30:59.748 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:30:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:59.771 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:30:59.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:30:59.835 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:30:59.872 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:30:59.895 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:30:59.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:30:59.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:30:59.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:30:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:30:59] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-09-21T10:30:59.996 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:00.018 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:00.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:00.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:00.033 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:31:00.120 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:00.153 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:00.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:00.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:00.195 INFO:tasks.radosbench.radosbench.0.smithi032.stdout: 89 16 116498 116482 81.782 11.375 0.00523491 0.0122222 2024-09-21T10:31:00.255 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:00 smithi045 ceph-mon[95976]: 1.3 scrub starts 2024-09-21T10:31:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:00 smithi045 ceph-mon[95976]: 1.3 scrub ok 2024-09-21T10:31:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:00 smithi045 ceph-mon[95976]: 1.0 scrub starts 2024-09-21T10:31:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:00 smithi045 ceph-mon[95976]: 1.0 scrub ok 2024-09-21T10:31:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:00 smithi045 ceph-mon[95976]: pgmap v1405: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s; 490/339204 objects degraded (0.144%); 75/339204 objects misplaced (0.022%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T10:31:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:00 smithi045 ceph-mon[95976]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T10:31:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:00 smithi045 ceph-mon[95976]: 1.12 scrub starts 2024-09-21T10:31:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:00 smithi045 ceph-mon[95976]: 1.12 scrub ok 2024-09-21T10:31:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:00 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1816252191' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:00.293 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:00.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:00.395 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:00.430 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:00.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:00.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[101299]: 1.3 scrub starts 2024-09-21T10:31:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[101299]: 1.3 scrub ok 2024-09-21T10:31:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[101299]: 1.0 scrub starts 2024-09-21T10:31:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[101299]: 1.0 scrub ok 2024-09-21T10:31:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[101299]: pgmap v1405: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s; 490/339204 objects degraded (0.144%); 75/339204 objects misplaced (0.022%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T10:31:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[101299]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T10:31:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[101299]: 1.12 scrub starts 2024-09-21T10:31:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[101299]: 1.12 scrub ok 2024-09-21T10:31:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1816252191' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[105748]: 1.3 scrub starts 2024-09-21T10:31:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[105748]: 1.3 scrub ok 2024-09-21T10:31:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[105748]: 1.0 scrub starts 2024-09-21T10:31:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[105748]: 1.0 scrub ok 2024-09-21T10:31:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[105748]: pgmap v1405: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s; 490/339204 objects degraded (0.144%); 75/339204 objects misplaced (0.022%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T10:31:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[105748]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T10:31:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[105748]: 1.12 scrub starts 2024-09-21T10:31:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[105748]: 1.12 scrub ok 2024-09-21T10:31:00.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:00 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1816252191' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:00.531 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:00.567 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:00.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:00.669 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:00.703 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:00.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:00.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:00.804 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:00.845 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:00.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:00.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:00.882 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 0, 4, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 6, 7, 5, 0] 2024-09-21T10:31:00.883 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:31:00.883 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [1, 2, 3, 4, 6, 7, 5, 0] 2024-09-21T10:31:00.883 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T10:31:00.883 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:31:00.909 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:31:00.947 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:00.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:00.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:00.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:01.076 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:01.110 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:01.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:01.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total time run: 90.0034 2024-09-21T10:31:01.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Total writes made: 117459 2024-09-21T10:31:01.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Write size: 65536 2024-09-21T10:31:01.196 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Object size: 65536 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Bandwidth (MB/sec): 81.5657 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Bandwidth: 43.1961 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max bandwidth (MB/sec): 236.125 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average IOPS: 1305 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev IOPS: 691.138 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max IOPS: 3778 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min IOPS: 0 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Average Latency(s): 0.0122561 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Stddev Latency(s): 0.03583 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Max latency(s): 2.23837 2024-09-21T10:31:01.197 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Min latency(s): 0.00102285 2024-09-21T10:31:01.200 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:31:01.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:00 smithi032 systemd[1]: Stopping Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:31:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[101299]: 1.d scrub starts 2024-09-21T10:31:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[101299]: 1.d scrub ok 2024-09-21T10:31:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:31:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1816252191' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:31:01.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[101299]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T10:31:01.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[105748]: 1.d scrub starts 2024-09-21T10:31:01.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[105748]: 1.d scrub ok 2024-09-21T10:31:01.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:31:01.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1816252191' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:31:01.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:01 smithi032 ceph-mon[105748]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T10:31:01.212 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:01.248 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:01.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:01.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:01 smithi045 ceph-mon[95976]: 1.d scrub starts 2024-09-21T10:31:01.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:01 smithi045 ceph-mon[95976]: 1.d scrub ok 2024-09-21T10:31:01.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:01 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:31:01.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:01 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1816252191' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:31:01.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:01 smithi045 ceph-mon[95976]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T10:31:01.350 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:01.389 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:01.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:01.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:01.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:01.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:01.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:01.634 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:01.671 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:01.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:01.772 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:01.809 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:01.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:01.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:01.911 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:01.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:01.947 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:01.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:02.048 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:02.086 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:02.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:02.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:02.187 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:02.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:02.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:02 smithi045 ceph-mon[95976]: 1.e scrub starts 2024-09-21T10:31:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:02 smithi045 ceph-mon[95976]: 1.e scrub ok 2024-09-21T10:31:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:02 smithi045 ceph-mon[95976]: pgmap v1408: 109 pgs: 2 peering, 107 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 8.2 MiB/s, 119 objects/s recovering 2024-09-21T10:31:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:02 smithi045 ceph-mon[95976]: 1.7 scrub starts 2024-09-21T10:31:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:02 smithi045 ceph-mon[95976]: 1.7 scrub ok 2024-09-21T10:31:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:02 smithi045 ceph-mon[95976]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 490/339204 objects degraded (0.144%), 2 pgs degraded) 2024-09-21T10:31:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:02 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:31:02.329 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:02.369 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:02.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:02.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[105748]: 1.e scrub starts 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[105748]: 1.e scrub ok 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[105748]: pgmap v1408: 109 pgs: 2 peering, 107 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 8.2 MiB/s, 119 objects/s recovering 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[105748]: 1.7 scrub starts 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[105748]: 1.7 scrub ok 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[105748]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 490/339204 objects degraded (0.144%), 2 pgs degraded) 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[101299]: 1.e scrub starts 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[101299]: 1.e scrub ok 2024-09-21T10:31:02.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[101299]: pgmap v1408: 109 pgs: 2 peering, 107 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 8.2 MiB/s, 119 objects/s recovering 2024-09-21T10:31:02.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[101299]: 1.7 scrub starts 2024-09-21T10:31:02.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[101299]: 1.7 scrub ok 2024-09-21T10:31:02.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[101299]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 490/339204 objects degraded (0.144%), 2 pgs degraded) 2024-09-21T10:31:02.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:02 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:31:02.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:02.509 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:02.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:02.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:02.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:02.658 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:02.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:02.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:02.760 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:02.800 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:02.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:02.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:02.902 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:02.939 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:02.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:03.041 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:03.078 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:03.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:03.180 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:03.220 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:03.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:03.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:03.321 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:03.362 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:03.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:03.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:03.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:03 smithi032 ceph-mon[101299]: 1.f scrub starts 2024-09-21T10:31:03.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:03 smithi032 ceph-mon[101299]: 1.f scrub ok 2024-09-21T10:31:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:03 smithi032 ceph-mon[105748]: 1.f scrub starts 2024-09-21T10:31:03.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:03 smithi032 ceph-mon[105748]: 1.f scrub ok 2024-09-21T10:31:03.464 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:03.500 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:03.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:03.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:03.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:03 smithi045 ceph-mon[95976]: 1.f scrub starts 2024-09-21T10:31:03.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:03 smithi045 ceph-mon[95976]: 1.f scrub ok 2024-09-21T10:31:03.601 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:03.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:03.646 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:03.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:03.746 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:03.779 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:03.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:03.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:03.880 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:03.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:03.927 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:03.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:04.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:04.072 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:04.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:04.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:04.173 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:04 smithi032 ceph-mon[105748]: pgmap v1409: 109 pgs: 2 peering, 107 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 43 MiB/s wr, 712 op/s; 6.4 MiB/s, 93 objects/s recovering 2024-09-21T10:31:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:04.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:04 smithi032 ceph-mon[101299]: pgmap v1409: 109 pgs: 2 peering, 107 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 43 MiB/s wr, 712 op/s; 6.4 MiB/s, 93 objects/s recovering 2024-09-21T10:31:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:04.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:04.210 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:04.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:04.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:04.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:04.348 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:04.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:04.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:04.450 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:04.488 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:04.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:04 smithi045 ceph-mon[95976]: pgmap v1409: 109 pgs: 2 peering, 107 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 43 MiB/s wr, 712 op/s; 6.4 MiB/s, 93 objects/s recovering 2024-09-21T10:31:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:04.589 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:04.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:04.625 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:04.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:04.726 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:04.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:04.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:04.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:04.884 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:04.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:04.938 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:04.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:05.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:05.076 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:05.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:05.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:05.178 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:05.215 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:05.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:05.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:05.317 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:05.347 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[271261]: 2024-09-21T10:31:05.245+0000 7ff59b9d7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:31:05.347 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[271261]: 2024-09-21T10:31:05.245+0000 7ff59b9d7640 -1 osd.0 454 *** Got signal Terminated *** 2024-09-21T10:31:05.347 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[271261]: 2024-09-21T10:31:05.245+0000 7ff59b9d7640 -1 osd.0 454 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:31:05.353 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:05.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:05.454 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:05.501 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:05.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:05.602 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:05.615 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:31:05.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:05.638 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:05.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:05.740 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:05.778 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:05.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:05.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:05.880 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:05.909 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:05.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:05.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:06.010 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:06.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:06.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:06.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:06.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:06.169 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:06.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:06.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:06.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:06 smithi032 ceph-mon[105748]: pgmap v1410: 109 pgs: 109 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 38 MiB/s wr, 870 op/s; 4.9 MiB/s, 72 objects/s recovering 2024-09-21T10:31:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:06 smithi032 ceph-mon[105748]: osd.0 marked itself down and dead 2024-09-21T10:31:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1125629690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:06 smithi032 ceph-mon[101299]: pgmap v1410: 109 pgs: 109 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 38 MiB/s wr, 870 op/s; 4.9 MiB/s, 72 objects/s recovering 2024-09-21T10:31:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:06 smithi032 ceph-mon[101299]: osd.0 marked itself down and dead 2024-09-21T10:31:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1125629690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:06.271 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:06.294 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:06.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:06.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:06.396 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:06.418 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:06.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:06.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:06.519 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:06.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:06 smithi045 ceph-mon[95976]: pgmap v1410: 109 pgs: 109 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 38 MiB/s wr, 870 op/s; 4.9 MiB/s, 72 objects/s recovering 2024-09-21T10:31:06.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:06 smithi045 ceph-mon[95976]: osd.0 marked itself down and dead 2024-09-21T10:31:06.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1125629690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:06.542 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:06.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:06.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:06.619 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:31:06.619 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:31:06.619 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:31:06.619 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:31:06.620 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:31:06.620 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:31:06.620 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:31:06.620 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:31:06.620 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:31:06.620 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:31:06.620 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:31:06.620 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:31:06.621 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:31:06.621 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:31:06.621 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:31:06.621 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:31:06.644 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:06.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:06.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:06.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:06.768 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:06.791 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:06.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:06.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:06.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:06.915 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:06.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:06.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:06.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:06 smithi032 podman[306966]: 2024-09-21 10:31:06.618518424 +0000 UTC m=+1.723994662 container died c09f6fabe76b2589964b26e712dbef9a1a02d300b50866157d1df75f498f6de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:31:07.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:07.039 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:07.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:07.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:07.140 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:07.167 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:07.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:07.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:07.269 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:07.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:07.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:07.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:07.333 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:07 smithi032 ceph-mon[101299]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:31:07.333 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:07 smithi032 ceph-mon[101299]: osdmap e455: 8 total, 7 up, 8 in 2024-09-21T10:31:07.333 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:07 smithi032 ceph-mon[105748]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:31:07.333 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:07 smithi032 ceph-mon[105748]: osdmap e455: 8 total, 7 up, 8 in 2024-09-21T10:31:07.414 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:07.452 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:07.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:07 smithi045 ceph-mon[95976]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:31:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:07 smithi045 ceph-mon[95976]: osdmap e455: 8 total, 7 up, 8 in 2024-09-21T10:31:07.554 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:07.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:07.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:07.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:07.628 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:07 smithi032 podman[306966]: 2024-09-21 10:31:07.331948641 +0000 UTC m=+2.437424874 container cleanup c09f6fabe76b2589964b26e712dbef9a1a02d300b50866157d1df75f498f6de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T10:31:07.628 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:07 smithi032 bash[306966]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:31:07.694 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:07.726 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:07.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:07.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:07.857 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:07.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:07.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:07.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:07 smithi032 podman[307179]: 2024-09-21 10:31:07.626419535 +0000 UTC m=+1.087683063 container remove c09f6fabe76b2589964b26e712dbef9a1a02d300b50866157d1df75f498f6de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T10:31:07.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:07.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:07.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:08.099 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:08.144 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:08.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:08.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:08.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:08.291 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:08.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:08.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:08.317 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:08 smithi032 ceph-mon[101299]: pgmap v1412: 109 pgs: 5 peering, 11 stale+active+clean, 93 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 36 MiB/s wr, 1.02k op/s; 4.9 MiB/s, 72 objects/s recovering 2024-09-21T10:31:08.317 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:08 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:31:08.317 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:08 smithi032 ceph-mon[101299]: osdmap e456: 8 total, 7 up, 8 in 2024-09-21T10:31:08.317 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:08 smithi032 ceph-mon[105748]: pgmap v1412: 109 pgs: 5 peering, 11 stale+active+clean, 93 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 36 MiB/s wr, 1.02k op/s; 4.9 MiB/s, 72 objects/s recovering 2024-09-21T10:31:08.317 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:08 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:31:08.317 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:08 smithi032 ceph-mon[105748]: osdmap e456: 8 total, 7 up, 8 in 2024-09-21T10:31:08.394 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:08.428 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:08.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:08.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:08.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:08.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:08 smithi045 ceph-mon[95976]: pgmap v1412: 109 pgs: 5 peering, 11 stale+active+clean, 93 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 36 MiB/s wr, 1.02k op/s; 4.9 MiB/s, 72 objects/s recovering 2024-09-21T10:31:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:08 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:31:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:08 smithi045 ceph-mon[95976]: osdmap e456: 8 total, 7 up, 8 in 2024-09-21T10:31:08.556 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:08.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:08.658 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:08.690 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:08.697 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:08.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:08.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:08.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:08 smithi032 podman[307259]: 2024-09-21 10:31:08.424470276 +0000 UTC m=+1.073435607 container died b465ee996cc77a4b3d1bc9f1a947b25611beb0af0f6d263b7a04bc87e146504c (image=quay.ceph.io/ceph-ci/ceph:reef, name=vibrant_curie, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2) 2024-09-21T10:31:08.798 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:08.826 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:08.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:08.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:08.959 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:08.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:08.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:09.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:09.097 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:09.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:09.198 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:09.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:09.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:09.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:09.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:09.372 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:09.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:09.474 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:09.502 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:09.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:09.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:09.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:09.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:09.699 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:31:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:31:09] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.51.0" 2024-09-21T10:31:09.734 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:09.767 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:09.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:09.869 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:09.902 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:09.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:10.004 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:10.034 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:10.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:10.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:10.135 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:10.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:31:10.175 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:10.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:10.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:10.277 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:10.307 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:10.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:10.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:10.322 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:10 smithi032 ceph-mon[101299]: pgmap v1414: 109 pgs: 5 peering, 11 stale+active+clean, 93 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 14 MiB/s wr, 652 op/s 2024-09-21T10:31:10.322 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:10 smithi032 ceph-mon[105748]: pgmap v1414: 109 pgs: 5 peering, 11 stale+active+clean, 93 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 14 MiB/s wr, 652 op/s 2024-09-21T10:31:10.408 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:10.443 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:10.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:10.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:10 smithi045 ceph-mon[95976]: pgmap v1414: 109 pgs: 5 peering, 11 stale+active+clean, 93 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 14 MiB/s wr, 652 op/s 2024-09-21T10:31:10.545 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:10.570 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:10.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:10.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:10.672 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:10.709 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:10.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:10.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:10.811 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:10.861 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:10.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:10.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:10.891 INFO:teuthology.orchestra.run.smithi032.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2024-09-21T10:31:10.962 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:11.007 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:11.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:11.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:11.109 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:11.149 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:11.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:11.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:11.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:11 smithi032 ceph-mon[105748]: Health check failed: Degraded data redundancy: 54773/330588 objects degraded (16.568%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:11.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:11 smithi032 ceph-mon[101299]: Health check failed: Degraded data redundancy: 54773/330588 objects degraded (16.568%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:11.251 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:11.282 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:11.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:11.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:11.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:11.411 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:11.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:11.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:11.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:11.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:11 smithi045 ceph-mon[95976]: Health check failed: Degraded data redundancy: 54773/330588 objects degraded (16.568%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:11.538 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:11.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:11.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:11.640 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:11.668 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:11.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:11.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:11.769 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:11.799 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:11.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:11.901 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:11.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:11.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:11.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:12.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:12.068 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:12.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:12.169 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:12.209 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:12.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:12.310 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:12.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:12 smithi032 ceph-mon[105748]: pgmap v1415: 109 pgs: 15 active+undersized, 5 peering, 4 stale+active+clean, 14 active+undersized+degraded, 71 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 6.1 MiB/s wr, 984 op/s; 54773/330588 objects degraded (16.568%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:12.346 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:12 smithi032 ceph-mon[101299]: pgmap v1415: 109 pgs: 15 active+undersized, 5 peering, 4 stale+active+clean, 14 active+undersized+degraded, 71 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 6.1 MiB/s wr, 984 op/s; 54773/330588 objects degraded (16.568%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:12.351 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:12.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:12.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:12.452 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:12.488 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:12.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:12.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:12 smithi045 ceph-mon[95976]: pgmap v1415: 109 pgs: 15 active+undersized, 5 peering, 4 stale+active+clean, 14 active+undersized+degraded, 71 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 6.1 MiB/s wr, 984 op/s; 54773/330588 objects degraded (16.568%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:12.590 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:12.626 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:12.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:12.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:12.727 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:12.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:12.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:12.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:12.866 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:12.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:12.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:12.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:12.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:12 smithi032 podman[307864]: 2024-09-21 10:31:12.670007869 +0000 UTC m=+0.110755734 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:31:12.997 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:13.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:13.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:13.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:13.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:13.136 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:31:13.151 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:31:13.152 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:31:13.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:13.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:13.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:13.229 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2178129802' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:31:13.229 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:13 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:31:13.229 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2178129802' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:31:13.229 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:13 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:31:13.273 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:13.303 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:13.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:13.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:13.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:13.445 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:13.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:13.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2178129802' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:31:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:13 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:31:13.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:13.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:13.592 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:13.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:13.594 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:13 smithi032 podman[307864]: 2024-09-21 10:31:13.227848402 +0000 UTC m=+0.668596266 container create a53c26343b011a07f7f77034c5c56e78427ea98f8162387d0ae663324f22198e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T10:31:13.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:13.721 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:13.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:13.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:13.823 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:13.863 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:13.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:13.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:13.879 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:13 smithi032 podman[307864]: 2024-09-21 10:31:13.724703018 +0000 UTC m=+1.165450876 container init a53c26343b011a07f7f77034c5c56e78427ea98f8162387d0ae663324f22198e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:31:13.879 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:13 smithi032 podman[307864]: 2024-09-21 10:31:13.729580241 +0000 UTC m=+1.170328105 container start a53c26343b011a07f7f77034c5c56e78427ea98f8162387d0ae663324f22198e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T10:31:13.887 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:13.948 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:13.964 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:13.999 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:14.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:14.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:14.101 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:14.128 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:14.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:14.133 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:13 smithi032 podman[307864]: 2024-09-21 10:31:13.874631313 +0000 UTC m=+1.315379189 container attach a53c26343b011a07f7f77034c5c56e78427ea98f8162387d0ae663324f22198e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:31:14.230 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:14.270 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:14.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:14.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:14.372 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:14.410 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:14.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:14.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:14.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[101299]: pgmap v1416: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 1.2 MiB/s wr, 783 op/s; 68305/328053 objects degraded (20.821%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:31:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-21T10:31:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:31:14.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[101299]: osdmap e457: 8 total, 7 up, 8 in 2024-09-21T10:31:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[105748]: pgmap v1416: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 1.2 MiB/s wr, 783 op/s; 68305/328053 objects degraded (20.821%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:31:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-21T10:31:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:31:14.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:14 smithi032 ceph-mon[105748]: osdmap e457: 8 total, 7 up, 8 in 2024-09-21T10:31:14.512 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:14 smithi045 ceph-mon[95976]: pgmap v1416: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 1.2 MiB/s wr, 783 op/s; 68305/328053 objects degraded (20.821%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:14 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:31:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:14 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-21T10:31:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:14 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:31:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:14 smithi045 ceph-mon[95976]: osdmap e457: 8 total, 7 up, 8 in 2024-09-21T10:31:14.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:14.543 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:14.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:14.644 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:14.676 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:14.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:14.778 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:14.806 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:14.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:14.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:14.908 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:14.945 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:14.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:14.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:15.047 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:15.078 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:15.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:15.180 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:15.223 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:15.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:15.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:15.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:15.350 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:15.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:15.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:15.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:15.499 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:15.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:15.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:15.601 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:15.641 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:15.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:15.742 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:15.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:15.783 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:15.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:15.884 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:15.910 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:15.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:16.012 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:16.040 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:16.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:16.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:16.142 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:16.186 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:16.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:16.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:16.288 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:16.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:16.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:16.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:16.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:16.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:16 smithi032 ceph-mon[101299]: pgmap v1418: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 66198/317310 objects degraded (20.862%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:16.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:16 smithi032 ceph-mon[105748]: pgmap v1418: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 66198/317310 objects degraded (20.862%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:16.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:31:16 smithi032 podman[307864]: 2024-09-21 10:31:16.108761918 +0000 UTC m=+3.549509783 container died a53c26343b011a07f7f77034c5c56e78427ea98f8162387d0ae663324f22198e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:31:16.472 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:16.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:16.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:16 smithi045 ceph-mon[95976]: pgmap v1418: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 66198/317310 objects degraded (20.862%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:16.573 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:16.600 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:16.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:16.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:16.702 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:16.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:16.742 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:16.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:16.843 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:16.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:16.884 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:16.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:16.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:17.023 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:17.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:17.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:17.125 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:17.154 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:17.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:17.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:17.256 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:17.288 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:17.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:17.390 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:17.404 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 66198/317310 objects degraded (20.862%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:17.404 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 66198/317310 objects degraded (20.862%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:17.417 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:17.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:17.503 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:31:17.520 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:17.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 66198/317310 objects degraded (20.862%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:17.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:17.559 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:17.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:17.573 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:31:17.573 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:31:17.573 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:31:17.573 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:31:17.573 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:31:17.573 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:31:17.573 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:31:17.574 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:31:17.574 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:31:17.574 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:31:17.574 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:31:17.574 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:31:17.574 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:31:17.574 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:31:17.574 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:31:17.659 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:17.696 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:17.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:17.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:17.798 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:17.837 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:17.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:17.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:17.938 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:17.973 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:17.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:17.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:18.074 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:18.118 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:18.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:18.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:18.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:18 smithi032 ceph-mon[105748]: pgmap v1419: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 64822/311079 objects degraded (20.838%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:18.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:18 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1502439136' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:18.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:18 smithi032 ceph-mon[101299]: pgmap v1419: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 64822/311079 objects degraded (20.838%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:18.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:18 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1502439136' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:18.219 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:18.251 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:18.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:18.353 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:18.398 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:18.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:18.500 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:18 smithi045 ceph-mon[95976]: pgmap v1419: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 64822/311079 objects degraded (20.838%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:18 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1502439136' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:18.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:18.536 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:18.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:18.637 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:18.664 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:18.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:18.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:18.766 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:18.801 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:18.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:18.829 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:31:18.829 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:31:18.903 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:18.932 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:18.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:18.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:19.034 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:19.062 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:19.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:19.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:19.164 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:19.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:19.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:19.302 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:19.308 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T10:31:19.332 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:19.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:19.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:19.387 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:31:19.387 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T10:31:19.433 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:19.475 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:19.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:19.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:19.541 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:31:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:31:19] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.51.0" 2024-09-21T10:31:19.576 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:19.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:19.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:19.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:19.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:19.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:19.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:19.853 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:19.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:19.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:19.885 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:19 smithi032 ceph-mon[101299]: pgmap v1420: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 64822/311079 objects degraded (20.838%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:19.886 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:19.886 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:19.886 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:19 smithi032 ceph-mon[105748]: pgmap v1420: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 64822/311079 objects degraded (20.838%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:19.890 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:19.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:19.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:19.933 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:31:19.933 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:31:19.992 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:20.025 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:20.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:20.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:19 smithi045 ceph-mon[95976]: pgmap v1420: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 64822/311079 objects degraded (20.838%); 0 B/s, 0 objects/s recovering 2024-09-21T10:31:20.127 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:20.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:20.168 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:20.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:20.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:20.297 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:20.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:20.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:20.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:20.438 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:20.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:20.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:20.540 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:20.577 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:20.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:20.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:20.673 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:31:20.678 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:20.690 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:20.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:20.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:20.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:20.819 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:20.830 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:31:20.856 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:20.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:20.958 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:20.994 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:20.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:21.096 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:21.123 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:21.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:21.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:21.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:21.257 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:21.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:21.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:21.359 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:21.384 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:21.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:21.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:21.486 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:21.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:21.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:21.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:21.619 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:21.658 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:21.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:21.759 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:21.788 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:21.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:21.890 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:21.916 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:21.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:21.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:22.017 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:22.063 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:22.064 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:22.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:22.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:22.068 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:22.166 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:22.191 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:21 smithi032 ceph-mon[105748]: pgmap v1421: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 62029/297456 objects degraded (20.853%) 2024-09-21T10:31:22.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:21 smithi032 ceph-mon[101299]: pgmap v1421: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 62029/297456 objects degraded (20.853%) 2024-09-21T10:31:22.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:22.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:22.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:21 smithi045 ceph-mon[95976]: pgmap v1421: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 62029/297456 objects degraded (20.853%) 2024-09-21T10:31:22.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:22.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:22.350 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:22.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:22.451 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:22.477 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:22.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:22.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:22.579 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:22.611 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:22.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:22.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:22.713 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:22.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:22.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:22.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:22.850 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:22.885 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:22.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:22.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:22.987 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:23.016 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:23.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:23.081 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 62029/297456 objects degraded (20.853%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:23.082 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 62029/297456 objects degraded (20.853%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:23.118 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:23.145 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:23.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:23.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:23.247 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:23.271 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:23.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:23.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:23.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 62029/297456 objects degraded (20.853%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:23.373 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:23.400 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:23.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:23.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:23.502 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:23.529 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:23.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:23.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:23.631 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:23.656 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:23.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:23.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:23.758 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:23.786 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:23.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:23.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:23.830 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:31:23.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:23.922 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:23.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:23.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:24.024 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:24.057 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:24.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:24.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:24.159 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:24.199 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:23 smithi032 ceph-mon[105748]: pgmap v1422: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 61322/294111 objects degraded (20.850%) 2024-09-21T10:31:24.200 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2733598169' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:24.200 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:23 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:24.200 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:23 smithi032 ceph-mon[101299]: pgmap v1422: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 61322/294111 objects degraded (20.850%) 2024-09-21T10:31:24.200 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2733598169' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:24.200 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:23 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:24.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:24.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:23 smithi045 ceph-mon[95976]: pgmap v1422: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 61322/294111 objects degraded (20.850%) 2024-09-21T10:31:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2733598169' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:23 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:24.302 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:24.335 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:24.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:24.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:24.388 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 0, 4, 1] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 6, 7, 5] 2024-09-21T10:31:24.388 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:31:24.388 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:31:24.389 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:31:24.437 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:24.479 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:24.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:24.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:24.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:24.622 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:24.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:24.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:24.724 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:24.762 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:24.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:24.864 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:24.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:24.904 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:24.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:25.005 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:25.030 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:25.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:25.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:25.072 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:24 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:31:25.073 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:24 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:31:25.073 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:24 smithi032 ceph-mon[105748]: osdmap e458: 8 total, 7 up, 8 in 2024-09-21T10:31:25.073 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:24 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:31:25.073 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:24 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:31:25.073 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:24 smithi032 ceph-mon[101299]: osdmap e458: 8 total, 7 up, 8 in 2024-09-21T10:31:25.132 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:25.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:25.171 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:25.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:25.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:25.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:24 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:31:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:24 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:31:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:24 smithi045 ceph-mon[95976]: osdmap e458: 8 total, 7 up, 8 in 2024-09-21T10:31:25.311 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:25.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:25.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:25.412 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:25.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:25.497 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:25.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:25.598 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:25.626 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:25.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:25.728 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:25.759 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:25.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:25.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:25.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:25.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:25.901 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:25.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:26.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:26.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:26.037 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:26.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:26.137 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:26.173 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:25 smithi032 ceph-mon[101299]: pgmap v1424: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 59250/283845 objects degraded (20.874%) 2024-09-21T10:31:26.173 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:25 smithi032 ceph-mon[105748]: pgmap v1424: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 59250/283845 objects degraded (20.874%) 2024-09-21T10:31:26.173 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:26.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:26.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:26.275 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:26.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:25 smithi045 ceph-mon[95976]: pgmap v1424: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 59250/283845 objects degraded (20.874%) 2024-09-21T10:31:26.310 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:26.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:26.412 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:26.446 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:26.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:26.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:26.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:26.588 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:26.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:26.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:26.690 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:26.722 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:26.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:26.821 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:31:26.824 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:26.854 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:31:26.854 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:31:26.867 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:26.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:26.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:26.892 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:31:26.892 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:31:26.916 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:31:26.917 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:31:26.956 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:31:26.956 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:31:26.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/939953326' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:26.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/939953326' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:26.968 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:26.995 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:31:26.996 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:31:26.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:26.998 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:26.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:27.029 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:31:27.030 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:31:27.069 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:31:27.069 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:31:27.092 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:31:27.092 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:31:27.093 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:31:27.093 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:31:27.093 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:31:27.098 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:27.107 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:31:27.136 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:27.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:27.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:27.237 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:27.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:27.269 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:27.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:27.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/939953326' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:27.370 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:27.414 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:27.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:27.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:27.515 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:27.550 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:27.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:27.651 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:27.680 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:27.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:27.781 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:27.820 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:27.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:27.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:27.922 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:27.962 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:27.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:28.063 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:28.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:28.106 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:28.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:28.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:28.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:27 smithi032 ceph-mon[101299]: pgmap v1425: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 57927/277650 objects degraded (20.863%) 2024-09-21T10:31:28.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:27 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 57927/277650 objects degraded (20.863%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:28.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:27 smithi032 ceph-mon[105748]: pgmap v1425: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 57927/277650 objects degraded (20.863%) 2024-09-21T10:31:28.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:27 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 57927/277650 objects degraded (20.863%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:28.238 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:28.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:28.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:27 smithi045 ceph-mon[95976]: pgmap v1425: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 57927/277650 objects degraded (20.863%) 2024-09-21T10:31:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:27 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 57927/277650 objects degraded (20.863%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:28.340 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:28.374 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:28.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:28.476 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:28.513 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:28.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:28.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:28.572 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:31:28.572 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:31:28.615 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:28.653 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:28.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:28.754 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:28.793 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:28.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:28.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:28.895 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:28.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:28.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:28.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:29.036 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:29.080 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:29.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:29.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:29.150 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:31:29.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:29.188 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:29.207 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:29.240 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:29.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:29.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:29.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:29.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:29.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:29.482 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:29.515 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:29.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:29.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:29.617 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:29.649 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:29.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:29.737 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:31:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:31:29] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.51.0" 2024-09-21T10:31:29.750 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:29.778 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:29.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:29.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:29.879 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:29.913 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:29.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:29.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:30.015 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:30.052 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:30.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:30.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:30.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:30.160 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:29 smithi032 ceph-mon[101299]: pgmap v1426: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 57927/277650 objects degraded (20.863%) 2024-09-21T10:31:30.161 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:29 smithi032 ceph-mon[105748]: pgmap v1426: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 57927/277650 objects degraded (20.863%) 2024-09-21T10:31:30.198 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:30.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:30.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:30.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:29 smithi045 ceph-mon[95976]: pgmap v1426: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 57927/277650 objects degraded (20.863%) 2024-09-21T10:31:30.300 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:30.343 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:30.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:30.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:30.445 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:30.486 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:30.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:30.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:30.574 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:31:30.587 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:30.623 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:30.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:30.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:30.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:30.761 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:30.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:30.863 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:30.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:30.909 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:30.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:31.010 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:31.043 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:31.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:31.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:31.145 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:31.173 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:31.182 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:31.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:31.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:31.284 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:31.316 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:31.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:31.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:31.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:31.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:31.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:31.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:31.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:31.593 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:31.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:31.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:31.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:31.725 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:31.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:31.827 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:31.855 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:31.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:31.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:31.956 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:31.983 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:31.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:31.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:32.019 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:31 smithi032 ceph-mon[105748]: pgmap v1427: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 55163/264378 objects degraded (20.865%) 2024-09-21T10:31:32.019 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:31 smithi032 ceph-mon[101299]: pgmap v1427: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 55163/264378 objects degraded (20.865%) 2024-09-21T10:31:32.084 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:32.115 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:32.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:32.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:32.217 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:32.257 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:32.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:31 smithi045 ceph-mon[95976]: pgmap v1427: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 55163/264378 objects degraded (20.865%) 2024-09-21T10:31:32.359 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:32.398 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:32.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:32.438 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:32.499 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:32.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:32.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:32.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:32.635 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:32.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:32.679 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:32.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:32.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:32.814 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:32.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:32.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:32.916 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:32.949 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:32.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:32.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:33.051 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:33.090 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:33.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:33.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:33.192 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:33.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:32 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 55163/264378 objects degraded (20.865%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:33.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:32 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 55163/264378 objects degraded (20.865%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:33.226 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:33.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:33.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:32 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 55163/264378 objects degraded (20.865%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:33.327 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:33.366 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:33.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:33.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:33.468 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:33.512 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:33.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:33.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:33.614 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:33.641 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:33.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:33.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:33.743 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:33.768 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:33.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:33.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:33.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:33.893 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:33.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:33.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:33.995 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:34.020 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:34.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:34.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:34.121 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:34.130 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:33 smithi032 ceph-mon[105748]: pgmap v1428: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 54443/261189 objects degraded (20.844%) 2024-09-21T10:31:34.130 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:34.131 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:33 smithi032 ceph-mon[101299]: pgmap v1428: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 54443/261189 objects degraded (20.844%) 2024-09-21T10:31:34.131 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:34.149 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:34.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:34.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:34.251 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:33 smithi045 ceph-mon[95976]: pgmap v1428: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 54443/261189 objects degraded (20.844%) 2024-09-21T10:31:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:34.286 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:34.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:34.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:34.388 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:34.415 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:34.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:34.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:34.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:34.567 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:34.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:34.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:34.669 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:34.696 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:34.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:34.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:34.798 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:34.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:34.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:34.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:34.868 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:31:34.942 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:34.978 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:34.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:35.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:35.113 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:35.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:35.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:35.135 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:34 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1446701809' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:31:35.136 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:34 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:31:35.136 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:34 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1446701809' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:31:35.136 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:34 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:31:35.215 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:35.245 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:35.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:35.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:35.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:34 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1446701809' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:31:35.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:34 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:31:35.307 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:31:35.312 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:31:35.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:35.373 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:35.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:35.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:35.475 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:35.502 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:35.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:35.603 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:35.632 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:35.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:35.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:35.733 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:35.767 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:35.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:35.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:35.868 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:35.895 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:35.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:35.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:35.996 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:36.033 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:36.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:36.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:36.134 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:36.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:36.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:36.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[101299]: pgmap v1429: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.32k op/s; 52387/250875 objects degraded (20.882%) 2024-09-21T10:31:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:31:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:31:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[101299]: osdmap e459: 8 total, 7 up, 8 in 2024-09-21T10:31:36.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[101299]: from='client.46304 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:31:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[105748]: pgmap v1429: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.32k op/s; 52387/250875 objects degraded (20.882%) 2024-09-21T10:31:36.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:31:36.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:31:36.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[105748]: osdmap e459: 8 total, 7 up, 8 in 2024-09-21T10:31:36.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:35 smithi032 ceph-mon[105748]: from='client.46304 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:31:36.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:35 smithi045 ceph-mon[95976]: pgmap v1429: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.32k op/s; 52387/250875 objects degraded (20.882%) 2024-09-21T10:31:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:35 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:31:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:35 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:31:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:35 smithi045 ceph-mon[95976]: osdmap e459: 8 total, 7 up, 8 in 2024-09-21T10:31:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:35 smithi045 ceph-mon[95976]: from='client.46304 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:31:36.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:36.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:36.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:36.414 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:36.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:36.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:36.549 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:36.577 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:36.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:36.679 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:36.727 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:36.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:36.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:36.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:36.855 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:36.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:36.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:36.898 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1430,"stamp":"2024-09-21T10:31:34.860006+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5414983677,"num_objects":82582,"num_object_clones":0,"num_object_copies":247746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":51692,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82582,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":153274,"num_write_kb":7530002,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1975,"num_bytes_recovered":132585131,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124718,"ondisk_log_size":124718,"up":285,"acting":285,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23549384,"kb_used_data":17356296,"kb_used_omap":45,"kb_used_meta":6192914,"kb_avail":726411832,"statfs":{"total":767960285184,"available":743845715968,"internally_reserved":0,"allocated":17772847104,"data_stored":17754473279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46417,"internal_metadata":6341544623},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,6,7,6,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-788594688,"num_objects":-12033,"num_object_clones":0,"num_object_copies":-36099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-7558,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-12033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12033,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.118483"},"pg_stats":[{"pgid":"1.11","version":"90'149","reported_seq":713,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.127162+0000","last_change":"2024-09-21T10:31:07.099238+0000","last_active":"2024-09-21T10:31:13.127162+0000","last_peered":"2024-09-21T10:31:13.127162+0000","last_clean":"2024-09-21T10:30:26.644096+0000","last_became_active":"2024-09-21T10:31:07.099238+0000","last_became_peered":"2024-09-21T10:31:07.099238+0000","last_unstale":"2024-09-21T10:31:13.127162+0000","last_undegraded":"2024-09-21T10:31:13.127162+0000","last_fullsized":"2024-09-21T10:31:07.096579+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:26.643960+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:30:26.643960+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:20.795582+0000","scrub_duration":0.013444612,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":810,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.829117+0000","last_change":"2024-09-21T10:30:45.210213+0000","last_active":"2024-09-21T10:31:23.829117+0000","last_peered":"2024-09-21T10:31:23.829117+0000","last_clean":"2024-09-21T10:31:23.829117+0000","last_became_active":"2024-09-21T10:30:45.207807+0000","last_became_peered":"2024-09-21T10:30:45.207807+0000","last_unstale":"2024-09-21T10:31:23.829117+0000","last_undegraded":"2024-09-21T10:31:23.829117+0000","last_fullsized":"2024-09-21T10:31:23.829117+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:00.839446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":464,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.128170+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:31:13.128170+0000","last_peered":"2024-09-21T10:31:13.128170+0000","last_clean":"2024-09-21T10:31:13.128170+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:31:13.128170+0000","last_undegraded":"2024-09-21T10:31:13.128170+0000","last_fullsized":"2024-09-21T10:31:13.128170+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'149","reported_seq":635,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.126870+0000","last_change":"2024-09-21T10:30:58.557091+0000","last_active":"2024-09-21T10:31:13.126870+0000","last_peered":"2024-09-21T10:31:13.126870+0000","last_clean":"2024-09-21T10:31:13.126870+0000","last_became_active":"2024-09-21T10:30:31.102167+0000","last_became_peered":"2024-09-21T10:30:31.102167+0000","last_unstale":"2024-09-21T10:31:13.126870+0000","last_undegraded":"2024-09-21T10:31:13.126870+0000","last_fullsized":"2024-09-21T10:31:13.126870+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:58.556988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:58.556988+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:55.249909+0000","scrub_duration":0.0013450059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":160,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.128828+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:31:13.128828+0000","last_peered":"2024-09-21T10:31:13.128828+0000","last_clean":"2024-09-21T10:31:13.128828+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:31:13.128828+0000","last_undegraded":"2024-09-21T10:31:13.128828+0000","last_fullsized":"2024-09-21T10:31:13.128828+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":39,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.128556+0000","last_change":"2024-09-21T10:31:07.104341+0000","last_active":"2024-09-21T10:31:13.128556+0000","last_peered":"2024-09-21T10:31:13.128556+0000","last_clean":"2024-09-21T10:12:59.519179+0000","last_became_active":"2024-09-21T10:31:07.104341+0000","last_became_peered":"2024-09-21T10:31:07.104341+0000","last_unstale":"2024-09-21T10:31:13.128556+0000","last_undegraded":"2024-09-21T10:31:13.128556+0000","last_fullsized":"2024-09-21T10:31:07.100124+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:18:24.422000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'149","reported_seq":778,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.827203+0000","last_change":"2024-09-21T10:30:47.984484+0000","last_active":"2024-09-21T10:31:23.827203+0000","last_peered":"2024-09-21T10:31:23.827203+0000","last_clean":"2024-09-21T10:31:23.827203+0000","last_became_active":"2024-09-21T10:30:47.981723+0000","last_became_peered":"2024-09-21T10:30:47.981723+0000","last_unstale":"2024-09-21T10:31:23.827203+0000","last_undegraded":"2024-09-21T10:31:23.827203+0000","last_fullsized":"2024-09-21T10:31:23.827203+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:26.687488+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:30:26.687488+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:23.843207+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":438,"reported_epoch":458,"state":"active+undersized","last_fresh":"2024-09-21T10:31:23.828941+0000","last_change":"2024-09-21T10:31:07.102180+0000","last_active":"2024-09-21T10:31:23.828941+0000","last_peered":"2024-09-21T10:31:23.828941+0000","last_clean":"2024-09-21T10:21:17.714149+0000","last_became_active":"2024-09-21T10:31:07.102180+0000","last_became_peered":"2024-09-21T10:31:07.102180+0000","last_unstale":"2024-09-21T10:31:23.828941+0000","last_undegraded":"2024-09-21T10:31:23.828941+0000","last_fullsized":"2024-09-21T10:31:07.097031+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:45:19.104397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":435,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.131315+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:31:13.131315+0000","last_peered":"2024-09-21T10:31:13.131315+0000","last_clean":"2024-09-21T10:31:13.131315+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:31:13.131315+0000","last_undegraded":"2024-09-21T10:31:13.131315+0000","last_fullsized":"2024-09-21T10:31:13.131315+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'149","reported_seq":669,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.126970+0000","last_change":"2024-09-21T10:30:59.599305+0000","last_active":"2024-09-21T10:31:13.126970+0000","last_peered":"2024-09-21T10:31:13.126970+0000","last_clean":"2024-09-21T10:31:13.126970+0000","last_became_active":"2024-09-21T10:30:47.960200+0000","last_became_peered":"2024-09-21T10:30:47.960200+0000","last_unstale":"2024-09-21T10:31:13.126970+0000","last_undegraded":"2024-09-21T10:31:13.126970+0000","last_fullsized":"2024-09-21T10:31:13.126970+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:59.599224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:59.599224+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:54.053533+0000","scrub_duration":0.001301359,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":824,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.834215+0000","last_change":"2024-09-21T10:30:45.209602+0000","last_active":"2024-09-21T10:31:23.834215+0000","last_peered":"2024-09-21T10:31:23.834215+0000","last_clean":"2024-09-21T10:31:23.834215+0000","last_became_active":"2024-09-21T10:30:45.209278+0000","last_became_peered":"2024-09-21T10:30:45.209278+0000","last_unstale":"2024-09-21T10:31:23.834215+0000","last_undegraded":"2024-09-21T10:31:23.834215+0000","last_fullsized":"2024-09-21T10:31:23.834215+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:35.045129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":537,"reported_epoch":457,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:13.134457+0000","last_change":"2024-09-21T10:31:07.106741+0000","last_active":"2024-09-21T10:31:13.134457+0000","last_peered":"2024-09-21T10:31:13.134457+0000","last_clean":"2024-09-21T10:31:00.035197+0000","last_became_active":"2024-09-21T10:31:07.106741+0000","last_became_peered":"2024-09-21T10:31:07.106741+0000","last_unstale":"2024-09-21T10:31:13.134457+0000","last_undegraded":"2024-09-21T10:31:07.097816+0000","last_fullsized":"2024-09-21T10:31:07.097698+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:50.079487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'149","reported_seq":671,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.131546+0000","last_change":"2024-09-21T10:30:59.559107+0000","last_active":"2024-09-21T10:31:13.131546+0000","last_peered":"2024-09-21T10:31:13.131546+0000","last_clean":"2024-09-21T10:31:13.131546+0000","last_became_active":"2024-09-21T10:30:47.965857+0000","last_became_peered":"2024-09-21T10:30:47.965857+0000","last_unstale":"2024-09-21T10:31:13.131546+0000","last_undegraded":"2024-09-21T10:31:13.131546+0000","last_fullsized":"2024-09-21T10:31:13.131546+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:59.559023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:59.559023+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:33.035344+0000","scrub_duration":0.0015473189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"458'9349","reported_seq":9401,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:31.145728+0000","last_change":"2024-09-21T10:31:07.104269+0000","last_active":"2024-09-21T10:31:31.145728+0000","last_peered":"2024-09-21T10:31:31.145728+0000","last_clean":"2024-09-21T10:31:06.020659+0000","last_became_active":"2024-09-21T10:31:07.104269+0000","last_became_peered":"2024-09-21T10:31:07.104269+0000","last_unstale":"2024-09-21T10:31:31.145728+0000","last_undegraded":"2024-09-21T10:31:07.098869+0000","last_fullsized":"2024-09-21T10:31:07.098745+0000","mapping_epoch":455,"log_start":"423'500","ondisk_log_start":"423'500","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":8849,"log_dups_size":0,"ondisk_log_size":8849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:31:47.298366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352256000,"num_objects":5375,"num_object_clones":0,"num_object_copies":16125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5375,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9349,"num_write_kb":471168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":5375}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":172,"reported_epoch":457,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:13.131863+0000","last_change":"2024-09-21T10:31:07.109861+0000","last_active":"2024-09-21T10:31:13.131863+0000","last_peered":"2024-09-21T10:31:13.131863+0000","last_clean":"2024-09-21T10:31:00.037354+0000","last_became_active":"2024-09-21T10:31:07.109861+0000","last_became_peered":"2024-09-21T10:31:07.109861+0000","last_unstale":"2024-09-21T10:31:13.131863+0000","last_undegraded":"2024-09-21T10:31:07.103223+0000","last_fullsized":"2024-09-21T10:31:07.103117+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:07.427134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":169,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.127039+0000","last_change":"2024-09-21T10:21:34.871942+0000","last_active":"2024-09-21T10:31:13.127039+0000","last_peered":"2024-09-21T10:31:13.127039+0000","last_clean":"2024-09-21T10:31:13.127039+0000","last_became_active":"2024-09-21T10:21:34.870527+0000","last_became_peered":"2024-09-21T10:21:34.870527+0000","last_unstale":"2024-09-21T10:31:13.127039+0000","last_undegraded":"2024-09-21T10:31:13.127039+0000","last_fullsized":"2024-09-21T10:31:13.127039+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:41.255135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"373'150","reported_seq":845,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:23.828843+0000","last_change":"2024-09-21T10:31:07.107574+0000","last_active":"2024-09-21T10:31:23.828843+0000","last_peered":"2024-09-21T10:31:23.828843+0000","last_clean":"2024-09-21T10:31:00.034841+0000","last_became_active":"2024-09-21T10:31:07.107574+0000","last_became_peered":"2024-09-21T10:31:07.107574+0000","last_unstale":"2024-09-21T10:31:23.828843+0000","last_undegraded":"2024-09-21T10:31:07.096670+0000","last_fullsized":"2024-09-21T10:31:07.096559+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:30:24.812285+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:30:24.812285+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:56.725046+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.7","version":"458'9411","reported_seq":9460,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:34.344646+0000","last_change":"2024-09-21T10:29:18.320824+0000","last_active":"2024-09-21T10:31:34.344646+0000","last_peered":"2024-09-21T10:31:34.344646+0000","last_clean":"2024-09-21T10:31:34.344646+0000","last_became_active":"2024-09-21T10:29:18.320520+0000","last_became_peered":"2024-09-21T10:29:18.320520+0000","last_unstale":"2024-09-21T10:31:34.344646+0000","last_undegraded":"2024-09-21T10:31:34.344646+0000","last_fullsized":"2024-09-21T10:31:34.344646+0000","mapping_epoch":419,"log_start":"424'2400","ondisk_log_start":"424'2400","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:23.749267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339279872,"num_objects":5177,"num_object_clones":0,"num_object_copies":15531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9411,"num_write_kb":466816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":164,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.132155+0000","last_change":"2024-09-21T10:30:45.225017+0000","last_active":"2024-09-21T10:31:13.132155+0000","last_peered":"2024-09-21T10:31:13.132155+0000","last_clean":"2024-09-21T10:31:13.132155+0000","last_became_active":"2024-09-21T10:30:45.214625+0000","last_became_peered":"2024-09-21T10:30:45.214625+0000","last_unstale":"2024-09-21T10:31:13.132155+0000","last_undegraded":"2024-09-21T10:31:13.132155+0000","last_fullsized":"2024-09-21T10:31:13.132155+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:41.994673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":175,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.126140+0000","last_change":"2024-09-21T10:30:45.206472+0000","last_active":"2024-09-21T10:31:13.126140+0000","last_peered":"2024-09-21T10:31:13.126140+0000","last_clean":"2024-09-21T10:31:13.126140+0000","last_became_active":"2024-09-21T10:30:45.205955+0000","last_became_peered":"2024-09-21T10:30:45.205955+0000","last_unstale":"2024-09-21T10:31:13.126140+0000","last_undegraded":"2024-09-21T10:31:13.126140+0000","last_fullsized":"2024-09-21T10:31:13.126140+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:50.262089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'149","reported_seq":581,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.135256+0000","last_change":"2024-09-21T10:31:01.909114+0000","last_active":"2024-09-21T10:31:13.135256+0000","last_peered":"2024-09-21T10:31:13.135256+0000","last_clean":"2024-09-21T10:31:13.135256+0000","last_became_active":"2024-09-21T10:30:31.109278+0000","last_became_peered":"2024-09-21T10:30:31.109278+0000","last_unstale":"2024-09-21T10:31:13.135256+0000","last_undegraded":"2024-09-21T10:31:13.135256+0000","last_fullsized":"2024-09-21T10:31:13.135256+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:31:01.905848+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:31:01.905848+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:16:19.880231+0000","scrub_duration":0.018313652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.4","version":"458'9684","reported_seq":9773,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:34.304474+0000","last_change":"2024-09-21T10:31:07.111184+0000","last_active":"2024-09-21T10:31:34.304474+0000","last_peered":"2024-09-21T10:31:34.304474+0000","last_clean":"2024-09-21T10:31:06.064375+0000","last_became_active":"2024-09-21T10:31:07.111184+0000","last_became_peered":"2024-09-21T10:31:07.111184+0000","last_unstale":"2024-09-21T10:31:34.304474+0000","last_undegraded":"2024-09-21T10:31:07.102140+0000","last_fullsized":"2024-09-21T10:31:07.101768+0000","mapping_epoch":455,"log_start":"426'3538","ondisk_log_start":"426'3538","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":6146,"log_dups_size":0,"ondisk_log_size":6146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:00.555401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335937536,"num_objects":5126,"num_object_clones":0,"num_object_copies":15378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5126,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9684,"num_write_kb":473920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":5126}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":460,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.132248+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:31:13.132248+0000","last_peered":"2024-09-21T10:31:13.132248+0000","last_clean":"2024-09-21T10:31:13.132248+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:31:13.132248+0000","last_undegraded":"2024-09-21T10:31:13.132248+0000","last_fullsized":"2024-09-21T10:31:13.132248+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":497,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.135860+0000","last_change":"2024-09-21T10:30:45.211258+0000","last_active":"2024-09-21T10:31:13.135860+0000","last_peered":"2024-09-21T10:31:13.135860+0000","last_clean":"2024-09-21T10:31:13.135860+0000","last_became_active":"2024-09-21T10:30:45.209507+0000","last_became_peered":"2024-09-21T10:30:45.209507+0000","last_unstale":"2024-09-21T10:31:13.135860+0000","last_undegraded":"2024-09-21T10:31:13.135860+0000","last_fullsized":"2024-09-21T10:31:13.135860+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:37.968652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'149","reported_seq":635,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.124702+0000","last_change":"2024-09-21T10:31:00.501007+0000","last_active":"2024-09-21T10:31:13.124702+0000","last_peered":"2024-09-21T10:31:13.124702+0000","last_clean":"2024-09-21T10:31:13.124702+0000","last_became_active":"2024-09-21T10:30:31.105952+0000","last_became_peered":"2024-09-21T10:30:31.105952+0000","last_unstale":"2024-09-21T10:31:13.124702+0000","last_undegraded":"2024-09-21T10:31:13.124702+0000","last_fullsized":"2024-09-21T10:31:13.124702+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:31:00.500754+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:31:00.500754+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:00.821304+0000","scrub_duration":0.0048757180000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.5","version":"458'9438","reported_seq":9531,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:34.793180+0000","last_change":"2024-09-21T10:31:07.102021+0000","last_active":"2024-09-21T10:31:34.793180+0000","last_peered":"2024-09-21T10:31:34.793180+0000","last_clean":"2024-09-21T10:31:06.089236+0000","last_became_active":"2024-09-21T10:31:07.102021+0000","last_became_peered":"2024-09-21T10:31:07.102021+0000","last_unstale":"2024-09-21T10:31:34.793180+0000","last_undegraded":"2024-09-21T10:31:07.098887+0000","last_fullsized":"2024-09-21T10:31:07.098765+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":9438,"log_dups_size":0,"ondisk_log_size":9438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:31:33.333410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330825728,"num_objects":5048,"num_object_clones":0,"num_object_copies":15144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5048,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9438,"num_write_kb":463552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":5048}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":394,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.828351+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:31:23.828351+0000","last_peered":"2024-09-21T10:31:23.828351+0000","last_clean":"2024-09-21T10:31:23.828351+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:31:23.828351+0000","last_undegraded":"2024-09-21T10:31:23.828351+0000","last_fullsized":"2024-09-21T10:31:23.828351+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":344,"reported_epoch":458,"state":"active+undersized","last_fresh":"2024-09-21T10:31:23.827887+0000","last_change":"2024-09-21T10:31:07.109248+0000","last_active":"2024-09-21T10:31:23.827887+0000","last_peered":"2024-09-21T10:31:23.827887+0000","last_clean":"2024-09-21T10:30:40.195902+0000","last_became_active":"2024-09-21T10:31:07.109248+0000","last_became_peered":"2024-09-21T10:31:07.109248+0000","last_unstale":"2024-09-21T10:31:23.827887+0000","last_undegraded":"2024-09-21T10:31:23.827887+0000","last_fullsized":"2024-09-21T10:31:07.101989+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:41.533279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"458'9377","reported_seq":10820,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:31.059656+0000","last_change":"2024-09-21T10:31:07.109301+0000","last_active":"2024-09-21T10:31:31.059656+0000","last_peered":"2024-09-21T10:31:31.059656+0000","last_clean":"2024-09-21T10:31:06.029631+0000","last_became_active":"2024-09-21T10:31:07.109301+0000","last_became_peered":"2024-09-21T10:31:07.109301+0000","last_unstale":"2024-09-21T10:31:31.059656+0000","last_undegraded":"2024-09-21T10:31:07.100692+0000","last_fullsized":"2024-09-21T10:31:07.100553+0000","mapping_epoch":455,"log_start":"424'2316","ondisk_log_start":"424'2316","created":419,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":7061,"log_dups_size":0,"ondisk_log_size":7061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:10:41.419727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351731712,"num_objects":5367,"num_object_clones":0,"num_object_copies":16101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5367,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9377,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":669,"num_bytes_recovered":43843584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":5367}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":745,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.130842+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:31:13.130842+0000","last_peered":"2024-09-21T10:31:13.130842+0000","last_clean":"2024-09-21T10:31:13.130842+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:31:13.130842+0000","last_undegraded":"2024-09-21T10:31:13.130842+0000","last_fullsized":"2024-09-21T10:31:13.130842+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":862,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:23.834918+0000","last_change":"2024-09-21T10:31:07.108437+0000","last_active":"2024-09-21T10:31:23.834918+0000","last_peered":"2024-09-21T10:31:23.834918+0000","last_clean":"2024-09-21T10:31:00.035207+0000","last_became_active":"2024-09-21T10:31:07.108437+0000","last_became_peered":"2024-09-21T10:31:07.108437+0000","last_unstale":"2024-09-21T10:31:23.834918+0000","last_undegraded":"2024-09-21T10:31:07.097079+0000","last_fullsized":"2024-09-21T10:31:07.096964+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:14.139271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":157,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.126066+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:31:13.126066+0000","last_peered":"2024-09-21T10:31:13.126066+0000","last_clean":"2024-09-21T10:31:13.126066+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:31:13.126066+0000","last_undegraded":"2024-09-21T10:31:13.126066+0000","last_fullsized":"2024-09-21T10:31:13.126066+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"458'9689","reported_seq":9779,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:34.789559+0000","last_change":"2024-09-21T10:29:18.322964+0000","last_active":"2024-09-21T10:31:34.789559+0000","last_peered":"2024-09-21T10:31:34.789559+0000","last_clean":"2024-09-21T10:31:34.789559+0000","last_became_active":"2024-09-21T10:29:18.322677+0000","last_became_peered":"2024-09-21T10:29:18.322677+0000","last_unstale":"2024-09-21T10:31:34.789559+0000","last_undegraded":"2024-09-21T10:31:34.789559+0000","last_fullsized":"2024-09-21T10:31:34.789559+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":9689,"log_dups_size":0,"ondisk_log_size":9689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:06.684046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332595200,"num_objects":5075,"num_object_clones":0,"num_object_copies":15225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9689,"num_write_kb":472448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":518,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.126959+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:31:13.126959+0000","last_peered":"2024-09-21T10:31:13.126959+0000","last_clean":"2024-09-21T10:31:13.126959+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:31:13.126959+0000","last_undegraded":"2024-09-21T10:31:13.126959+0000","last_fullsized":"2024-09-21T10:31:13.126959+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":487,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.135946+0000","last_change":"2024-09-21T10:30:45.223044+0000","last_active":"2024-09-21T10:31:13.135946+0000","last_peered":"2024-09-21T10:31:13.135946+0000","last_clean":"2024-09-21T10:31:13.135946+0000","last_became_active":"2024-09-21T10:30:45.222166+0000","last_became_peered":"2024-09-21T10:30:45.222166+0000","last_unstale":"2024-09-21T10:31:13.135946+0000","last_undegraded":"2024-09-21T10:31:13.135946+0000","last_fullsized":"2024-09-21T10:31:13.135946+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:16.215392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":175,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.127823+0000","last_change":"2024-09-21T10:30:45.206465+0000","last_active":"2024-09-21T10:31:13.127823+0000","last_peered":"2024-09-21T10:31:13.127823+0000","last_clean":"2024-09-21T10:31:13.127823+0000","last_became_active":"2024-09-21T10:30:45.206088+0000","last_became_peered":"2024-09-21T10:30:45.206088+0000","last_unstale":"2024-09-21T10:31:13.127823+0000","last_undegraded":"2024-09-21T10:31:13.127823+0000","last_fullsized":"2024-09-21T10:31:13.127823+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:46:01.557325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'149","reported_seq":802,"reported_epoch":458,"state":"active+undersized","last_fresh":"2024-09-21T10:31:23.827576+0000","last_change":"2024-09-21T10:31:07.104125+0000","last_active":"2024-09-21T10:31:23.827576+0000","last_peered":"2024-09-21T10:31:23.827576+0000","last_clean":"2024-09-21T10:31:00.037665+0000","last_became_active":"2024-09-21T10:31:07.104125+0000","last_became_peered":"2024-09-21T10:31:07.104125+0000","last_unstale":"2024-09-21T10:31:23.827576+0000","last_undegraded":"2024-09-21T10:31:23.827576+0000","last_fullsized":"2024-09-21T10:31:07.101244+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:24.704596+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:30:24.704596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:30.315683+0000","scrub_duration":14,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"458'9510","reported_seq":10540,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:34.380949+0000","last_change":"2024-09-21T10:31:00.043848+0000","last_active":"2024-09-21T10:31:34.380949+0000","last_peered":"2024-09-21T10:31:34.380949+0000","last_clean":"2024-09-21T10:31:34.380949+0000","last_became_active":"2024-09-21T10:31:00.038067+0000","last_became_peered":"2024-09-21T10:31:00.038067+0000","last_unstale":"2024-09-21T10:31:34.380949+0000","last_undegraded":"2024-09-21T10:31:34.380949+0000","last_fullsized":"2024-09-21T10:31:34.380949+0000","mapping_epoch":453,"log_start":"424'2500","ondisk_log_start":"424'2500","created":419,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:31.816028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332529664,"num_objects":5074,"num_object_clones":0,"num_object_copies":15222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9510,"num_write_kb":466688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":478,"num_bytes_recovered":31326208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":157,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.125345+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:31:13.125345+0000","last_peered":"2024-09-21T10:31:13.125345+0000","last_clean":"2024-09-21T10:31:13.125345+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:31:13.125345+0000","last_undegraded":"2024-09-21T10:31:13.125345+0000","last_fullsized":"2024-09-21T10:31:13.125345+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":175,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.124980+0000","last_change":"2024-09-21T10:30:45.220586+0000","last_active":"2024-09-21T10:31:13.124980+0000","last_peered":"2024-09-21T10:31:13.124980+0000","last_clean":"2024-09-21T10:31:13.124980+0000","last_became_active":"2024-09-21T10:30:45.219727+0000","last_became_peered":"2024-09-21T10:30:45.219727+0000","last_unstale":"2024-09-21T10:31:13.124980+0000","last_undegraded":"2024-09-21T10:31:13.124980+0000","last_fullsized":"2024-09-21T10:31:13.124980+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:42.454658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"458'9376","reported_seq":9430,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:34.304653+0000","last_change":"2024-09-21T10:31:07.111977+0000","last_active":"2024-09-21T10:31:34.304653+0000","last_peered":"2024-09-21T10:31:34.304653+0000","last_clean":"2024-09-21T10:31:06.091631+0000","last_became_active":"2024-09-21T10:31:07.111977+0000","last_became_peered":"2024-09-21T10:31:07.111977+0000","last_unstale":"2024-09-21T10:31:34.304653+0000","last_undegraded":"2024-09-21T10:31:07.101893+0000","last_fullsized":"2024-09-21T10:31:07.101776+0000","mapping_epoch":455,"log_start":"426'3240","ondisk_log_start":"426'3240","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":6136,"log_dups_size":0,"ondisk_log_size":6136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:22:44.313921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328466432,"num_objects":5012,"num_object_clones":0,"num_object_copies":15036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5012,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9376,"num_write_kb":460416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":5012}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":696,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.126877+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:31:13.126877+0000","last_peered":"2024-09-21T10:31:13.126877+0000","last_clean":"2024-09-21T10:31:13.126877+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:31:13.126877+0000","last_undegraded":"2024-09-21T10:31:13.126877+0000","last_fullsized":"2024-09-21T10:31:13.126877+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":839,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.827800+0000","last_change":"2024-09-21T10:30:45.207431+0000","last_active":"2024-09-21T10:31:23.827800+0000","last_peered":"2024-09-21T10:31:23.827800+0000","last_clean":"2024-09-21T10:31:23.827800+0000","last_became_active":"2024-09-21T10:30:45.207075+0000","last_became_peered":"2024-09-21T10:30:45.207075+0000","last_unstale":"2024-09-21T10:31:23.827800+0000","last_undegraded":"2024-09-21T10:31:23.827800+0000","last_fullsized":"2024-09-21T10:31:23.827800+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:08.684578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":40,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.828618+0000","last_change":"2024-09-21T10:30:45.210139+0000","last_active":"2024-09-21T10:31:23.828618+0000","last_peered":"2024-09-21T10:31:23.828618+0000","last_clean":"2024-09-21T10:31:23.828618+0000","last_became_active":"2024-09-21T10:30:45.205910+0000","last_became_peered":"2024-09-21T10:30:45.205910+0000","last_unstale":"2024-09-21T10:31:23.828618+0000","last_undegraded":"2024-09-21T10:31:23.828618+0000","last_fullsized":"2024-09-21T10:31:23.828618+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:42.109452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"458'9511","reported_seq":10565,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:34.398865+0000","last_change":"2024-09-21T10:31:07.147782+0000","last_active":"2024-09-21T10:31:34.398865+0000","last_peered":"2024-09-21T10:31:34.398865+0000","last_clean":"2024-09-21T10:31:06.089621+0000","last_became_active":"2024-09-21T10:31:07.119767+0000","last_became_peered":"2024-09-21T10:31:07.119767+0000","last_unstale":"2024-09-21T10:31:34.398865+0000","last_undegraded":"2024-09-21T10:31:07.110476+0000","last_fullsized":"2024-09-21T10:31:07.098190+0000","mapping_epoch":455,"log_start":"424'2500","ondisk_log_start":"424'2500","created":419,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:20:00.053377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334430208,"num_objects":5103,"num_object_clones":0,"num_object_copies":15309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5103,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9511,"num_write_kb":467648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":490,"num_bytes_recovered":31916032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":5103}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":771,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.128400+0000","last_change":"2024-09-21T10:31:07.115582+0000","last_active":"2024-09-21T10:31:13.128400+0000","last_peered":"2024-09-21T10:31:13.128400+0000","last_clean":"2024-09-21T10:31:00.036604+0000","last_became_active":"2024-09-21T10:31:07.115582+0000","last_became_peered":"2024-09-21T10:31:07.115582+0000","last_unstale":"2024-09-21T10:31:13.128400+0000","last_undegraded":"2024-09-21T10:31:13.128400+0000","last_fullsized":"2024-09-21T10:31:07.100074+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:58.578099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:58.578099+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:22.545859+0000","scrub_duration":0.016223530999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":773,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.128476+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:31:13.128476+0000","last_peered":"2024-09-21T10:31:13.128476+0000","last_clean":"2024-09-21T10:31:13.128476+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:31:13.128476+0000","last_undegraded":"2024-09-21T10:31:13.128476+0000","last_fullsized":"2024-09-21T10:31:13.128476+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":757,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.829291+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:31:23.829291+0000","last_peered":"2024-09-21T10:31:23.829291+0000","last_clean":"2024-09-21T10:31:23.829291+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:31:23.829291+0000","last_undegraded":"2024-09-21T10:31:23.829291+0000","last_fullsized":"2024-09-21T10:31:23.829291+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":436,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.135545+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:31:13.135545+0000","last_peered":"2024-09-21T10:31:13.135545+0000","last_clean":"2024-09-21T10:31:13.135545+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:31:13.135545+0000","last_undegraded":"2024-09-21T10:31:13.135545+0000","last_fullsized":"2024-09-21T10:31:13.135545+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"458'9816","reported_seq":9905,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:34.750002+0000","last_change":"2024-09-21T10:31:07.102383+0000","last_active":"2024-09-21T10:31:34.750002+0000","last_peered":"2024-09-21T10:31:34.750002+0000","last_clean":"2024-09-21T10:31:06.039724+0000","last_became_active":"2024-09-21T10:31:07.102383+0000","last_became_peered":"2024-09-21T10:31:07.102383+0000","last_unstale":"2024-09-21T10:31:34.750002+0000","last_undegraded":"2024-09-21T10:31:07.098949+0000","last_fullsized":"2024-09-21T10:31:07.098834+0000","mapping_epoch":455,"log_start":"424'814","ondisk_log_start":"424'814","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":9002,"log_dups_size":0,"ondisk_log_size":9002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:32.815631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341704704,"num_objects":5214,"num_object_clones":0,"num_object_copies":15642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5214,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9816,"num_write_kb":480960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":5214}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":634,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.828057+0000","last_change":"2024-09-21T10:30:58.540050+0000","last_active":"2024-09-21T10:31:23.828057+0000","last_peered":"2024-09-21T10:31:23.828057+0000","last_clean":"2024-09-21T10:31:23.828057+0000","last_became_active":"2024-09-21T10:30:47.975931+0000","last_became_peered":"2024-09-21T10:30:47.975931+0000","last_unstale":"2024-09-21T10:31:23.828057+0000","last_undegraded":"2024-09-21T10:31:23.828057+0000","last_fullsized":"2024-09-21T10:31:23.828057+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:58.539975+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:30:58.539975+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:34.963092+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1209,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:23.835997+0000","last_change":"2024-09-21T10:31:07.105245+0000","last_active":"2024-09-21T10:31:23.835997+0000","last_peered":"2024-09-21T10:31:23.835997+0000","last_clean":"2024-09-21T10:31:00.039774+0000","last_became_active":"2024-09-21T10:31:07.105245+0000","last_became_peered":"2024-09-21T10:31:07.105245+0000","last_unstale":"2024-09-21T10:31:23.835997+0000","last_undegraded":"2024-09-21T10:31:07.101787+0000","last_fullsized":"2024-09-21T10:31:07.101646+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:37.999585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":218,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.126206+0000","last_change":"2024-09-21T10:31:07.111181+0000","last_active":"2024-09-21T10:31:13.126206+0000","last_peered":"2024-09-21T10:31:13.126206+0000","last_clean":"2024-09-21T10:19:39.754468+0000","last_became_active":"2024-09-21T10:31:07.111181+0000","last_became_peered":"2024-09-21T10:31:07.111181+0000","last_unstale":"2024-09-21T10:31:13.126206+0000","last_undegraded":"2024-09-21T10:31:13.126206+0000","last_fullsized":"2024-09-21T10:31:07.101414+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:28.988270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":153,"reported_epoch":458,"state":"active+undersized","last_fresh":"2024-09-21T10:31:23.831589+0000","last_change":"2024-09-21T10:31:07.103711+0000","last_active":"2024-09-21T10:31:23.831589+0000","last_peered":"2024-09-21T10:31:23.831589+0000","last_clean":"2024-09-21T10:30:40.195902+0000","last_became_active":"2024-09-21T10:31:07.103711+0000","last_became_peered":"2024-09-21T10:31:07.103711+0000","last_unstale":"2024-09-21T10:31:23.831589+0000","last_undegraded":"2024-09-21T10:31:23.831589+0000","last_fullsized":"2024-09-21T10:31:07.100363+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:13.591483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"458'9544","reported_seq":9590,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:31.189273+0000","last_change":"2024-09-21T10:29:18.320919+0000","last_active":"2024-09-21T10:31:31.189273+0000","last_peered":"2024-09-21T10:31:31.189273+0000","last_clean":"2024-09-21T10:31:31.189273+0000","last_became_active":"2024-09-21T10:29:18.320597+0000","last_became_peered":"2024-09-21T10:29:18.320597+0000","last_unstale":"2024-09-21T10:31:31.189273+0000","last_undegraded":"2024-09-21T10:31:31.189273+0000","last_fullsized":"2024-09-21T10:31:31.189273+0000","mapping_epoch":419,"log_start":"423'700","ondisk_log_start":"423'700","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":8844,"log_dups_size":0,"ondisk_log_size":8844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:30.147686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353107968,"num_objects":5388,"num_object_clones":0,"num_object_copies":16164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9544,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":787,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.128688+0000","last_change":"2024-09-21T10:31:07.109094+0000","last_active":"2024-09-21T10:31:13.128688+0000","last_peered":"2024-09-21T10:31:13.128688+0000","last_clean":"2024-09-21T10:31:00.036719+0000","last_became_active":"2024-09-21T10:31:07.109094+0000","last_became_peered":"2024-09-21T10:31:07.109094+0000","last_unstale":"2024-09-21T10:31:13.128688+0000","last_undegraded":"2024-09-21T10:31:13.128688+0000","last_fullsized":"2024-09-21T10:31:07.100566+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:25.635933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:25.635933+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:35.042430+0000","scrub_duration":0.010887529,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1361,"reported_epoch":458,"state":"active+undersized","last_fresh":"2024-09-21T10:31:23.833996+0000","last_change":"2024-09-21T10:31:07.104030+0000","last_active":"2024-09-21T10:31:23.833996+0000","last_peered":"2024-09-21T10:31:23.833996+0000","last_clean":"2024-09-21T10:31:00.045617+0000","last_became_active":"2024-09-21T10:31:07.104030+0000","last_became_peered":"2024-09-21T10:31:07.104030+0000","last_unstale":"2024-09-21T10:31:23.833996+0000","last_undegraded":"2024-09-21T10:31:23.833996+0000","last_fullsized":"2024-09-21T10:31:07.099659+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:58.667781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":40,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.827969+0000","last_change":"2024-09-21T10:30:45.211483+0000","last_active":"2024-09-21T10:31:23.827969+0000","last_peered":"2024-09-21T10:31:23.827969+0000","last_clean":"2024-09-21T10:31:23.827969+0000","last_became_active":"2024-09-21T10:30:45.211153+0000","last_became_peered":"2024-09-21T10:30:45.211153+0000","last_unstale":"2024-09-21T10:31:23.827969+0000","last_undegraded":"2024-09-21T10:31:23.827969+0000","last_fullsized":"2024-09-21T10:31:23.827969+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:47:17.004820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":44,"reported_epoch":458,"state":"active+undersized","last_fresh":"2024-09-21T10:31:23.828469+0000","last_change":"2024-09-21T10:31:07.105687+0000","last_active":"2024-09-21T10:31:23.828469+0000","last_peered":"2024-09-21T10:31:23.828469+0000","last_clean":"2024-09-21T10:31:00.038185+0000","last_became_active":"2024-09-21T10:31:07.105687+0000","last_became_peered":"2024-09-21T10:31:07.105687+0000","last_unstale":"2024-09-21T10:31:23.828469+0000","last_undegraded":"2024-09-21T10:31:23.828469+0000","last_fullsized":"2024-09-21T10:31:07.100012+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:59.641394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"458'9582","reported_seq":9636,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:34.397571+0000","last_change":"2024-09-21T10:31:07.114174+0000","last_active":"2024-09-21T10:31:34.397571+0000","last_peered":"2024-09-21T10:31:34.397571+0000","last_clean":"2024-09-21T10:31:06.083866+0000","last_became_active":"2024-09-21T10:31:07.114174+0000","last_became_peered":"2024-09-21T10:31:07.114174+0000","last_unstale":"2024-09-21T10:31:34.397571+0000","last_undegraded":"2024-09-21T10:31:07.103084+0000","last_fullsized":"2024-09-21T10:31:07.098001+0000","mapping_epoch":455,"log_start":"425'2600","ondisk_log_start":"425'2600","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:13:32.857690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338558976,"num_objects":5166,"num_object_clones":0,"num_object_copies":15498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5166,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9582,"num_write_kb":471936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":5166}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":738,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.127305+0000","last_change":"2024-09-21T10:31:07.100817+0000","last_active":"2024-09-21T10:31:13.127305+0000","last_peered":"2024-09-21T10:31:13.127305+0000","last_clean":"2024-09-21T10:31:00.034561+0000","last_became_active":"2024-09-21T10:31:07.100817+0000","last_became_peered":"2024-09-21T10:31:07.100817+0000","last_unstale":"2024-09-21T10:31:13.127305+0000","last_undegraded":"2024-09-21T10:31:13.127305+0000","last_fullsized":"2024-09-21T10:31:07.096435+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:22.717550+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:30:22.717550+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:50.495810+0000","scrub_duration":0.0073004899999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1290,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.127407+0000","last_change":"2024-09-21T10:30:51.982312+0000","last_active":"2024-09-21T10:31:13.127407+0000","last_peered":"2024-09-21T10:31:13.127407+0000","last_clean":"2024-09-21T10:31:13.127407+0000","last_became_active":"2024-09-21T10:30:51.981405+0000","last_became_peered":"2024-09-21T10:30:51.981405+0000","last_unstale":"2024-09-21T10:31:13.127407+0000","last_undegraded":"2024-09-21T10:31:13.127407+0000","last_fullsized":"2024-09-21T10:31:13.127407+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:15.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":178,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.125143+0000","last_change":"2024-09-21T10:31:07.115953+0000","last_active":"2024-09-21T10:31:13.125143+0000","last_peered":"2024-09-21T10:31:13.125143+0000","last_clean":"2024-09-21T10:31:00.036571+0000","last_became_active":"2024-09-21T10:31:07.115953+0000","last_became_peered":"2024-09-21T10:31:07.115953+0000","last_unstale":"2024-09-21T10:31:13.125143+0000","last_undegraded":"2024-09-21T10:31:13.125143+0000","last_fullsized":"2024-09-21T10:31:07.105324+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:29:42.137618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":40,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.827721+0000","last_change":"2024-09-21T10:30:45.210421+0000","last_active":"2024-09-21T10:31:23.827721+0000","last_peered":"2024-09-21T10:31:23.827721+0000","last_clean":"2024-09-21T10:31:23.827721+0000","last_became_active":"2024-09-21T10:30:45.206024+0000","last_became_peered":"2024-09-21T10:30:45.206024+0000","last_unstale":"2024-09-21T10:31:23.827721+0000","last_undegraded":"2024-09-21T10:31:23.827721+0000","last_fullsized":"2024-09-21T10:31:23.827721+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:51.844704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"458'9488","reported_seq":9541,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:34.388606+0000","last_change":"2024-09-21T10:31:07.109973+0000","last_active":"2024-09-21T10:31:34.388606+0000","last_peered":"2024-09-21T10:31:34.388606+0000","last_clean":"2024-09-21T10:31:06.040916+0000","last_became_active":"2024-09-21T10:31:07.109973+0000","last_became_peered":"2024-09-21T10:31:07.109973+0000","last_unstale":"2024-09-21T10:31:34.388606+0000","last_undegraded":"2024-09-21T10:31:07.100831+0000","last_fullsized":"2024-09-21T10:31:07.100713+0000","mapping_epoch":455,"log_start":"424'2300","ondisk_log_start":"424'2300","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":7188,"log_dups_size":0,"ondisk_log_size":7188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:23:36.707020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334888960,"num_objects":5110,"num_object_clones":0,"num_object_copies":15330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5110,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9488,"num_write_kb":467136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":5110}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":565,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.135163+0000","last_change":"2024-09-21T10:31:00.915456+0000","last_active":"2024-09-21T10:31:13.135163+0000","last_peered":"2024-09-21T10:31:13.135163+0000","last_clean":"2024-09-21T10:31:13.135163+0000","last_became_active":"2024-09-21T10:30:22.053430+0000","last_became_peered":"2024-09-21T10:30:22.053430+0000","last_unstale":"2024-09-21T10:31:13.135163+0000","last_undegraded":"2024-09-21T10:31:13.135163+0000","last_fullsized":"2024-09-21T10:31:13.135163+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:31:00.915274+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:31:00.915274+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:09.913485+0000","scrub_duration":0.0051508819999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":409,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.128677+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:31:13.128677+0000","last_peered":"2024-09-21T10:31:13.128677+0000","last_clean":"2024-09-21T10:31:13.128677+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:31:13.128677+0000","last_undegraded":"2024-09-21T10:31:13.128677+0000","last_fullsized":"2024-09-21T10:31:13.128677+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":777,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:23.827445+0000","last_change":"2024-09-21T10:31:07.104363+0000","last_active":"2024-09-21T10:31:23.827445+0000","last_peered":"2024-09-21T10:31:23.827445+0000","last_clean":"2024-09-21T10:31:00.037550+0000","last_became_active":"2024-09-21T10:31:07.104363+0000","last_became_peered":"2024-09-21T10:31:07.104363+0000","last_unstale":"2024-09-21T10:31:23.827445+0000","last_undegraded":"2024-09-21T10:31:07.100737+0000","last_fullsized":"2024-09-21T10:31:07.100626+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:01.648577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":40,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.827298+0000","last_change":"2024-09-21T10:30:45.223025+0000","last_active":"2024-09-21T10:31:23.827298+0000","last_peered":"2024-09-21T10:31:23.827298+0000","last_clean":"2024-09-21T10:31:23.827298+0000","last_became_active":"2024-09-21T10:30:45.222585+0000","last_became_peered":"2024-09-21T10:30:45.222585+0000","last_unstale":"2024-09-21T10:31:23.827298+0000","last_undegraded":"2024-09-21T10:31:23.827298+0000","last_fullsized":"2024-09-21T10:31:23.827298+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:14.096008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"458'9570","reported_seq":9622,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:34.402702+0000","last_change":"2024-09-21T10:31:07.112311+0000","last_active":"2024-09-21T10:31:34.402702+0000","last_peered":"2024-09-21T10:31:34.402702+0000","last_clean":"2024-09-21T10:31:06.039883+0000","last_became_active":"2024-09-21T10:31:07.112311+0000","last_became_peered":"2024-09-21T10:31:07.112311+0000","last_unstale":"2024-09-21T10:31:34.402702+0000","last_undegraded":"2024-09-21T10:31:07.097126+0000","last_fullsized":"2024-09-21T10:31:07.097003+0000","mapping_epoch":455,"log_start":"424'2500","ondisk_log_start":"424'2500","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":7070,"log_dups_size":0,"ondisk_log_size":7070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:12.750608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337969176,"num_objects":5158,"num_object_clones":0,"num_object_copies":15474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5158,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5158,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9570,"num_write_kb":471233,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":5158}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":586,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.136245+0000","last_change":"2024-09-21T10:31:07.102401+0000","last_active":"2024-09-21T10:31:13.136245+0000","last_peered":"2024-09-21T10:31:13.136245+0000","last_clean":"2024-09-21T10:31:00.035822+0000","last_became_active":"2024-09-21T10:31:07.102401+0000","last_became_peered":"2024-09-21T10:31:07.102401+0000","last_unstale":"2024-09-21T10:31:13.136245+0000","last_undegraded":"2024-09-21T10:31:13.136245+0000","last_fullsized":"2024-09-21T10:31:07.096930+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:26.906418+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:26.906418+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:46.823715+0000","scrub_duration":0.0087083520000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":40,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.827638+0000","last_change":"2024-09-21T10:30:45.210705+0000","last_active":"2024-09-21T10:31:23.827638+0000","last_peered":"2024-09-21T10:31:23.827638+0000","last_clean":"2024-09-21T10:31:23.827638+0000","last_became_active":"2024-09-21T10:30:45.205528+0000","last_became_peered":"2024-09-21T10:30:45.205528+0000","last_unstale":"2024-09-21T10:31:23.827638+0000","last_undegraded":"2024-09-21T10:31:23.827638+0000","last_fullsized":"2024-09-21T10:31:23.827638+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:32.515942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":1049,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.131740+0000","last_change":"2024-09-21T10:31:07.108652+0000","last_active":"2024-09-21T10:31:13.131740+0000","last_peered":"2024-09-21T10:31:13.131740+0000","last_clean":"2024-09-21T10:31:00.037132+0000","last_became_active":"2024-09-21T10:31:07.108652+0000","last_became_peered":"2024-09-21T10:31:07.108652+0000","last_unstale":"2024-09-21T10:31:13.131740+0000","last_undegraded":"2024-09-21T10:31:13.131740+0000","last_fullsized":"2024-09-21T10:31:07.102634+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:30.017414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":354,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.128562+0000","last_change":"2024-09-21T10:31:07.101404+0000","last_active":"2024-09-21T10:31:13.128562+0000","last_peered":"2024-09-21T10:31:13.128562+0000","last_clean":"2024-09-21T10:17:25.114921+0000","last_became_active":"2024-09-21T10:31:07.101404+0000","last_became_peered":"2024-09-21T10:31:07.101404+0000","last_unstale":"2024-09-21T10:31:13.128562+0000","last_undegraded":"2024-09-21T10:31:13.128562+0000","last_fullsized":"2024-09-21T10:31:07.098631+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:42:01.229293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"458'9629","reported_seq":9677,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:34.313460+0000","last_change":"2024-09-21T10:29:18.321928+0000","last_active":"2024-09-21T10:31:34.313460+0000","last_peered":"2024-09-21T10:31:34.313460+0000","last_clean":"2024-09-21T10:31:34.313460+0000","last_became_active":"2024-09-21T10:29:18.321303+0000","last_became_peered":"2024-09-21T10:29:18.321303+0000","last_unstale":"2024-09-21T10:31:34.313460+0000","last_undegraded":"2024-09-21T10:31:34.313460+0000","last_fullsized":"2024-09-21T10:31:34.313460+0000","mapping_epoch":419,"log_start":"426'3506","ondisk_log_start":"426'3506","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":6123,"log_dups_size":0,"ondisk_log_size":6123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:25.083294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339148800,"num_objects":5175,"num_object_clones":0,"num_object_copies":15525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9629,"num_write_kb":473728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":717,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.127894+0000","last_change":"2024-09-21T10:30:22.584185+0000","last_active":"2024-09-21T10:31:13.127894+0000","last_peered":"2024-09-21T10:31:13.127894+0000","last_clean":"2024-09-21T10:31:13.127894+0000","last_became_active":"2024-09-21T10:30:22.056768+0000","last_became_peered":"2024-09-21T10:30:22.056768+0000","last_unstale":"2024-09-21T10:31:13.127894+0000","last_undegraded":"2024-09-21T10:31:13.127894+0000","last_fullsized":"2024-09-21T10:31:13.127894+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:22.583825+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:22.583825+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:59.427627+0000","scrub_duration":0.0051540479999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":845,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:23.834510+0000","last_change":"2024-09-21T10:31:07.103303+0000","last_active":"2024-09-21T10:31:23.834510+0000","last_peered":"2024-09-21T10:31:23.834510+0000","last_clean":"2024-09-21T10:31:00.035566+0000","last_became_active":"2024-09-21T10:31:07.103303+0000","last_became_peered":"2024-09-21T10:31:07.103303+0000","last_unstale":"2024-09-21T10:31:23.834510+0000","last_undegraded":"2024-09-21T10:31:07.097965+0000","last_fullsized":"2024-09-21T10:31:07.097863+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:13.764165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":699,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.831827+0000","last_change":"2024-09-21T10:30:51.986197+0000","last_active":"2024-09-21T10:31:23.831827+0000","last_peered":"2024-09-21T10:31:23.831827+0000","last_clean":"2024-09-21T10:31:23.831827+0000","last_became_active":"2024-09-21T10:30:51.985466+0000","last_became_peered":"2024-09-21T10:30:51.985466+0000","last_unstale":"2024-09-21T10:31:23.831827+0000","last_undegraded":"2024-09-21T10:31:23.831827+0000","last_fullsized":"2024-09-21T10:31:23.831827+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:16.999338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":841,"reported_epoch":458,"state":"active+undersized","last_fresh":"2024-09-21T10:31:23.826796+0000","last_change":"2024-09-21T10:31:07.104938+0000","last_active":"2024-09-21T10:31:23.826796+0000","last_peered":"2024-09-21T10:31:23.826796+0000","last_clean":"2024-09-21T10:31:00.034510+0000","last_became_active":"2024-09-21T10:31:07.104938+0000","last_became_peered":"2024-09-21T10:31:07.104938+0000","last_unstale":"2024-09-21T10:31:23.826796+0000","last_undegraded":"2024-09-21T10:31:23.826796+0000","last_fullsized":"2024-09-21T10:31:07.096148+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:08:16.199220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"458'9392","reported_seq":9440,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:34.399460+0000","last_change":"2024-09-21T10:29:18.318036+0000","last_active":"2024-09-21T10:31:34.399460+0000","last_peered":"2024-09-21T10:31:34.399460+0000","last_clean":"2024-09-21T10:31:34.399460+0000","last_became_active":"2024-09-21T10:29:18.317716+0000","last_became_peered":"2024-09-21T10:29:18.317716+0000","last_unstale":"2024-09-21T10:31:34.399460+0000","last_undegraded":"2024-09-21T10:31:34.399460+0000","last_fullsized":"2024-09-21T10:31:34.399460+0000","mapping_epoch":419,"log_start":"424'2400","ondisk_log_start":"424'2400","created":419,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:29:17.303121+0000","last_clean_scrub_stamp":"2024-09-21T10:29:17.303121+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:58.738776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326762496,"num_objects":4986,"num_object_clones":0,"num_object_copies":14958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9392,"num_write_kb":460096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":827,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.826765+0000","last_change":"2024-09-21T10:30:22.757965+0000","last_active":"2024-09-21T10:31:23.826765+0000","last_peered":"2024-09-21T10:31:23.826765+0000","last_clean":"2024-09-21T10:31:23.826765+0000","last_became_active":"2024-09-21T10:30:22.041873+0000","last_became_peered":"2024-09-21T10:30:22.041873+0000","last_unstale":"2024-09-21T10:31:23.826765+0000","last_undegraded":"2024-09-21T10:31:23.826765+0000","last_fullsized":"2024-09-21T10:31:23.826765+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:30:22.757772+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:30:22.757772+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:50.549712+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":186,"reported_epoch":458,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:23.828258+0000","last_change":"2024-09-21T10:31:07.109245+0000","last_active":"2024-09-21T10:31:23.828258+0000","last_peered":"2024-09-21T10:31:23.828258+0000","last_clean":"2024-09-21T10:30:40.196110+0000","last_became_active":"2024-09-21T10:31:07.109245+0000","last_became_peered":"2024-09-21T10:31:07.109245+0000","last_unstale":"2024-09-21T10:31:23.828258+0000","last_undegraded":"2024-09-21T10:31:07.100860+0000","last_fullsized":"2024-09-21T10:31:07.100754+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:29.390415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":771,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.135352+0000","last_change":"2024-09-21T10:24:16.515108+0000","last_active":"2024-09-21T10:31:13.135352+0000","last_peered":"2024-09-21T10:31:13.135352+0000","last_clean":"2024-09-21T10:31:13.135352+0000","last_became_active":"2024-09-21T10:21:34.865731+0000","last_became_peered":"2024-09-21T10:21:34.865731+0000","last_unstale":"2024-09-21T10:31:13.135352+0000","last_undegraded":"2024-09-21T10:31:13.135352+0000","last_fullsized":"2024-09-21T10:31:13.135352+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:01.686954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":433,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.126270+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:31:13.126270+0000","last_peered":"2024-09-21T10:31:13.126270+0000","last_clean":"2024-09-21T10:31:13.126270+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:31:13.126270+0000","last_undegraded":"2024-09-21T10:31:13.126270+0000","last_fullsized":"2024-09-21T10:31:13.126270+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":475,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.125258+0000","last_change":"2024-09-21T10:30:45.220621+0000","last_active":"2024-09-21T10:31:13.125258+0000","last_peered":"2024-09-21T10:31:13.125258+0000","last_clean":"2024-09-21T10:31:13.125258+0000","last_became_active":"2024-09-21T10:30:45.219593+0000","last_became_peered":"2024-09-21T10:30:45.219593+0000","last_unstale":"2024-09-21T10:31:13.125258+0000","last_undegraded":"2024-09-21T10:31:13.125258+0000","last_fullsized":"2024-09-21T10:31:13.125258+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:27:44.914386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":167,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.833502+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:31:23.833502+0000","last_peered":"2024-09-21T10:31:23.833502+0000","last_clean":"2024-09-21T10:31:23.833502+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:31:23.833502+0000","last_undegraded":"2024-09-21T10:31:23.833502+0000","last_fullsized":"2024-09-21T10:31:23.833502+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":40,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.827458+0000","last_change":"2024-09-21T10:30:45.217613+0000","last_active":"2024-09-21T10:31:23.827458+0000","last_peered":"2024-09-21T10:31:23.827458+0000","last_clean":"2024-09-21T10:31:23.827458+0000","last_became_active":"2024-09-21T10:30:45.217188+0000","last_became_peered":"2024-09-21T10:30:45.217188+0000","last_unstale":"2024-09-21T10:31:23.827458+0000","last_undegraded":"2024-09-21T10:31:23.827458+0000","last_fullsized":"2024-09-21T10:31:23.827458+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:54.177397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":140,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:14.371583+0000","last_change":"2024-09-21T10:21:34.879712+0000","last_active":"2024-09-21T10:31:14.371583+0000","last_peered":"2024-09-21T10:31:14.371583+0000","last_clean":"2024-09-21T10:31:14.371583+0000","last_became_active":"2024-09-21T10:21:34.863165+0000","last_became_peered":"2024-09-21T10:21:34.863165+0000","last_unstale":"2024-09-21T10:31:14.371583+0000","last_undegraded":"2024-09-21T10:31:14.371583+0000","last_fullsized":"2024-09-21T10:31:14.371583+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:42:03.934515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"373'150","reported_seq":830,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.833905+0000","last_change":"2024-09-21T10:30:31.115487+0000","last_active":"2024-09-21T10:31:23.833905+0000","last_peered":"2024-09-21T10:31:23.833905+0000","last_clean":"2024-09-21T10:31:23.833905+0000","last_became_active":"2024-09-21T10:30:31.114941+0000","last_became_peered":"2024-09-21T10:30:31.114941+0000","last_unstale":"2024-09-21T10:31:23.833905+0000","last_undegraded":"2024-09-21T10:31:23.833905+0000","last_fullsized":"2024-09-21T10:31:23.833905+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:30:23.771955+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:30:23.771955+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:20:04.739094+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":186,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.128404+0000","last_change":"2024-09-21T10:30:45.199098+0000","last_active":"2024-09-21T10:31:13.128404+0000","last_peered":"2024-09-21T10:31:13.128404+0000","last_clean":"2024-09-21T10:31:13.128404+0000","last_became_active":"2024-09-21T10:30:45.198762+0000","last_became_peered":"2024-09-21T10:30:45.198762+0000","last_unstale":"2024-09-21T10:31:13.128404+0000","last_undegraded":"2024-09-21T10:31:13.128404+0000","last_fullsized":"2024-09-21T10:31:13.128404+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:40.566015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":749,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.827232+0000","last_change":"2024-09-21T10:21:34.863993+0000","last_active":"2024-09-21T10:31:23.827232+0000","last_peered":"2024-09-21T10:31:23.827232+0000","last_clean":"2024-09-21T10:31:23.827232+0000","last_became_active":"2024-09-21T10:21:34.863655+0000","last_became_peered":"2024-09-21T10:21:34.863655+0000","last_unstale":"2024-09-21T10:31:23.827232+0000","last_undegraded":"2024-09-21T10:31:23.827232+0000","last_fullsized":"2024-09-21T10:31:23.827232+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:55.405427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":231,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.129354+0000","last_change":"2024-09-21T10:31:07.099872+0000","last_active":"2024-09-21T10:31:13.129354+0000","last_peered":"2024-09-21T10:31:13.129354+0000","last_clean":"2024-09-21T10:17:25.115497+0000","last_became_active":"2024-09-21T10:31:07.099872+0000","last_became_peered":"2024-09-21T10:31:07.099872+0000","last_unstale":"2024-09-21T10:31:13.129354+0000","last_undegraded":"2024-09-21T10:31:13.129354+0000","last_fullsized":"2024-09-21T10:31:07.098437+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:33.112573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":823,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.826691+0000","last_change":"2024-09-21T10:30:45.207106+0000","last_active":"2024-09-21T10:31:23.826691+0000","last_peered":"2024-09-21T10:31:23.826691+0000","last_clean":"2024-09-21T10:31:23.826691+0000","last_became_active":"2024-09-21T10:30:45.206347+0000","last_became_peered":"2024-09-21T10:30:45.206347+0000","last_unstale":"2024-09-21T10:31:23.826691+0000","last_undegraded":"2024-09-21T10:31:23.826691+0000","last_fullsized":"2024-09-21T10:31:23.826691+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:42.646798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":493,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.135246+0000","last_change":"2024-09-21T10:30:45.209630+0000","last_active":"2024-09-21T10:31:13.135246+0000","last_peered":"2024-09-21T10:31:13.135246+0000","last_clean":"2024-09-21T10:31:13.135246+0000","last_became_active":"2024-09-21T10:30:45.209304+0000","last_became_peered":"2024-09-21T10:30:45.209304+0000","last_unstale":"2024-09-21T10:31:13.135246+0000","last_undegraded":"2024-09-21T10:31:13.135246+0000","last_fullsized":"2024-09-21T10:31:13.135246+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:29.693394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":855,"reported_epoch":458,"state":"active+undersized","last_fresh":"2024-09-21T10:31:23.826979+0000","last_change":"2024-09-21T10:31:07.101662+0000","last_active":"2024-09-21T10:31:23.826979+0000","last_peered":"2024-09-21T10:31:23.826979+0000","last_clean":"2024-09-21T10:31:00.034601+0000","last_became_active":"2024-09-21T10:31:07.101662+0000","last_became_peered":"2024-09-21T10:31:07.101662+0000","last_unstale":"2024-09-21T10:31:23.826979+0000","last_undegraded":"2024-09-21T10:31:23.826979+0000","last_fullsized":"2024-09-21T10:31:07.096209+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:03.313018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":264,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.132042+0000","last_change":"2024-09-21T10:31:07.104313+0000","last_active":"2024-09-21T10:31:13.132042+0000","last_peered":"2024-09-21T10:31:13.132042+0000","last_clean":"2024-09-21T10:21:17.754673+0000","last_became_active":"2024-09-21T10:31:07.104313+0000","last_became_peered":"2024-09-21T10:31:07.104313+0000","last_unstale":"2024-09-21T10:31:13.132042+0000","last_undegraded":"2024-09-21T10:31:13.132042+0000","last_fullsized":"2024-09-21T10:31:07.099697+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:28.669303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":40,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.826960+0000","last_change":"2024-09-21T10:30:45.206647+0000","last_active":"2024-09-21T10:31:23.826960+0000","last_peered":"2024-09-21T10:31:23.826960+0000","last_clean":"2024-09-21T10:31:23.826960+0000","last_became_active":"2024-09-21T10:30:45.205723+0000","last_became_peered":"2024-09-21T10:30:45.205723+0000","last_unstale":"2024-09-21T10:31:23.826960+0000","last_undegraded":"2024-09-21T10:31:23.826960+0000","last_fullsized":"2024-09-21T10:31:23.826960+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:55:33.281876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":830,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.835802+0000","last_change":"2024-09-21T10:30:45.207518+0000","last_active":"2024-09-21T10:31:23.835802+0000","last_peered":"2024-09-21T10:31:23.835802+0000","last_clean":"2024-09-21T10:31:23.835802+0000","last_became_active":"2024-09-21T10:30:45.207075+0000","last_became_peered":"2024-09-21T10:30:45.207075+0000","last_unstale":"2024-09-21T10:31:23.835802+0000","last_undegraded":"2024-09-21T10:31:23.835802+0000","last_fullsized":"2024-09-21T10:31:23.835802+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:33.239764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":482,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.125544+0000","last_change":"2024-09-21T10:30:45.222126+0000","last_active":"2024-09-21T10:31:13.125544+0000","last_peered":"2024-09-21T10:31:13.125544+0000","last_clean":"2024-09-21T10:31:13.125544+0000","last_became_active":"2024-09-21T10:30:45.221747+0000","last_became_peered":"2024-09-21T10:30:45.221747+0000","last_unstale":"2024-09-21T10:31:13.125544+0000","last_undegraded":"2024-09-21T10:31:13.125544+0000","last_fullsized":"2024-09-21T10:31:13.125544+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:15.856857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":829,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.834069+0000","last_change":"2024-09-21T10:30:45.216426+0000","last_active":"2024-09-21T10:31:23.834069+0000","last_peered":"2024-09-21T10:31:23.834069+0000","last_clean":"2024-09-21T10:31:23.834069+0000","last_became_active":"2024-09-21T10:30:45.215768+0000","last_became_peered":"2024-09-21T10:30:45.215768+0000","last_unstale":"2024-09-21T10:31:23.834069+0000","last_undegraded":"2024-09-21T10:31:23.834069+0000","last_fullsized":"2024-09-21T10:31:23.834069+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:13.389909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":62,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.125003+0000","last_change":"2024-09-21T10:31:07.111451+0000","last_active":"2024-09-21T10:31:13.125003+0000","last_peered":"2024-09-21T10:31:13.125003+0000","last_clean":"2024-09-21T10:21:59.010058+0000","last_became_active":"2024-09-21T10:31:07.111451+0000","last_became_peered":"2024-09-21T10:31:07.111451+0000","last_unstale":"2024-09-21T10:31:13.125003+0000","last_undegraded":"2024-09-21T10:31:13.125003+0000","last_fullsized":"2024-09-21T10:31:07.105327+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:19:30.373567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":174,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.124905+0000","last_change":"2024-09-21T10:30:45.220820+0000","last_active":"2024-09-21T10:31:13.124905+0000","last_peered":"2024-09-21T10:31:13.124905+0000","last_clean":"2024-09-21T10:31:13.124905+0000","last_became_active":"2024-09-21T10:30:45.220126+0000","last_became_peered":"2024-09-21T10:30:45.220126+0000","last_unstale":"2024-09-21T10:31:13.124905+0000","last_undegraded":"2024-09-21T10:31:13.124905+0000","last_fullsized":"2024-09-21T10:31:13.124905+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:31.832090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":426,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.128367+0000","last_change":"2024-09-21T10:30:45.220768+0000","last_active":"2024-09-21T10:31:13.128367+0000","last_peered":"2024-09-21T10:31:13.128367+0000","last_clean":"2024-09-21T10:31:13.128367+0000","last_became_active":"2024-09-21T10:30:45.214059+0000","last_became_peered":"2024-09-21T10:30:45.214059+0000","last_unstale":"2024-09-21T10:31:13.128367+0000","last_undegraded":"2024-09-21T10:31:13.128367+0000","last_fullsized":"2024-09-21T10:31:13.128367+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:12:52.858780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":177,"reported_epoch":457,"state":"active+undersized","last_fresh":"2024-09-21T10:31:13.125360+0000","last_change":"2024-09-21T10:31:07.108896+0000","last_active":"2024-09-21T10:31:13.125360+0000","last_peered":"2024-09-21T10:31:13.125360+0000","last_clean":"2024-09-21T10:31:00.036710+0000","last_became_active":"2024-09-21T10:31:07.108896+0000","last_became_peered":"2024-09-21T10:31:07.108896+0000","last_unstale":"2024-09-21T10:31:13.125360+0000","last_undegraded":"2024-09-21T10:31:13.125360+0000","last_fullsized":"2024-09-21T10:31:07.101245+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:36:05.954068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":378,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.128756+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:31:13.128756+0000","last_peered":"2024-09-21T10:31:13.128756+0000","last_clean":"2024-09-21T10:31:13.128756+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:31:13.128756+0000","last_undegraded":"2024-09-21T10:31:13.128756+0000","last_fullsized":"2024-09-21T10:31:13.128756+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":844,"reported_epoch":458,"state":"active+undersized","last_fresh":"2024-09-21T10:31:23.827633+0000","last_change":"2024-09-21T10:31:07.101627+0000","last_active":"2024-09-21T10:31:23.827633+0000","last_peered":"2024-09-21T10:31:23.827633+0000","last_clean":"2024-09-21T10:31:00.034663+0000","last_became_active":"2024-09-21T10:31:07.101627+0000","last_became_peered":"2024-09-21T10:31:07.101627+0000","last_unstale":"2024-09-21T10:31:23.827633+0000","last_undegraded":"2024-09-21T10:31:23.827633+0000","last_fullsized":"2024-09-21T10:31:07.096605+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:59.463483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":791,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-21T10:31:23.829107+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:31:23.829107+0000","last_peered":"2024-09-21T10:31:23.829107+0000","last_clean":"2024-09-21T10:31:23.829107+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:31:23.829107+0000","last_undegraded":"2024-09-21T10:31:23.829107+0000","last_fullsized":"2024-09-21T10:31:23.829107+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":174,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-21T10:31:13.125472+0000","last_change":"2024-09-21T10:30:45.206962+0000","last_active":"2024-09-21T10:31:13.125472+0000","last_peered":"2024-09-21T10:31:13.125472+0000","last_clean":"2024-09-21T10:31:13.125472+0000","last_became_active":"2024-09-21T10:30:45.206174+0000","last_became_peered":"2024-09-21T10:30:45.206174+0000","last_unstale":"2024-09-21T10:31:13.125472+0000","last_undegraded":"2024-09-21T10:31:13.125472+0000","last_fullsized":"2024-09-21T10:31:13.125472+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:03:00.246788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":72,"reported_epoch":457,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:31:13.129227+0000","last_change":"2024-09-21T10:31:07.102632+0000","last_active":"2024-09-21T10:31:13.129227+0000","last_peered":"2024-09-21T10:31:13.129227+0000","last_clean":"2024-09-21T10:21:58.990990+0000","last_became_active":"2024-09-21T10:31:07.102632+0000","last_became_peered":"2024-09-21T10:31:07.102632+0000","last_unstale":"2024-09-21T10:31:13.129227+0000","last_undegraded":"2024-09-21T10:31:07.099009+0000","last_fullsized":"2024-09-21T10:31:07.098889+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:01.071311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5410193432,"num_objects":82554,"num_object_clones":0,"num_object_copies":247662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":51679,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82554,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152366,"num_write_kb":7517377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1637,"num_bytes_recovered":107085824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17612943360,"data_stored":17612931144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120552,"ondisk_log_size":120552,"up":38,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":87,"acting":87,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":5746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28818,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":84,"acting":84,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":278,"num_bytes_recovered":25165881,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":20,"acting":20,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3133,"ondisk_log_size":3133,"up":56,"acting":56,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234503,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2259592,"kb_used_data":1335936,"kb_used_omap":1,"kb_used_meta":923646,"kb_avail":91485560,"statfs":{"total":95995035648,"available":93681213440,"internally_reserved":0,"allocated":1367998464,"data_stored":1365693033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":945813963},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496141,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3182152,"kb_used_data":2320832,"kb_used_omap":1,"kb_used_meta":861310,"kb_avail":90563000,"statfs":{"total":95995035648,"available":92736512000,"internally_reserved":0,"allocated":2376531968,"data_stored":2374226765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":881981895},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569404040,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2323840,"kb_used_data":1736504,"kb_used_omap":7,"kb_used_meta":587320,"kb_avail":91421312,"statfs":{"total":95995035648,"available":93615423488,"internally_reserved":0,"allocated":1778180096,"data_stored":1775879431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":601416600},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019414,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3466316,"kb_used_data":2646852,"kb_used_omap":1,"kb_used_meta":819454,"kb_avail":90278836,"statfs":{"total":95995035648,"available":92445528064,"internally_reserved":0,"allocated":2710376448,"data_stored":2708067206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":839121350},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435598,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2688632,"kb_used_data":1628208,"kb_used_omap":5,"kb_used_meta":1060410,"kb_avail":91056520,"statfs":{"total":95995035648,"available":93241876480,"internally_reserved":0,"allocated":1667284992,"data_stored":1664991950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":1085859880},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599125,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2847352,"kb_used_data":1967408,"kb_used_omap":7,"kb_used_meta":879928,"kb_avail":90897800,"statfs":{"total":95995035648,"available":93079347200,"internally_reserved":0,"allocated":2014625792,"data_stored":2012332968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":901046733},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,3,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762651,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1884976,"kb_used_data":1041064,"kb_used_omap":7,"kb_used_meta":843832,"kb_avail":91860176,"statfs":{"total":95995035648,"available":94064820224,"internally_reserved":0,"allocated":1066049536,"data_stored":1063763798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":864084221},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":389,"seq":1670742278208,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4896524,"kb_used_data":4679492,"kb_used_omap":12,"kb_used_meta":217011,"kb_avail":88848628,"statfs":{"total":95995035648,"available":90980995072,"internally_reserved":0,"allocated":4791799808,"data_stored":4789518128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12595,"internal_metadata":222219981},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11001,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4760145920,"data_stored":4760141848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1047265280,"data_stored":1047265280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2005860352,"data_stored":2005860352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1659240448,"data_stored":1659240448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2685014016,"data_stored":2685009944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1761148928,"data_stored":1761148928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2348355584,"data_stored":2348351512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1345912832,"data_stored":1345912832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:31:36.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 21 2024-09-21T10:31:36.957 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:36.984 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:36.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:37.085 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:37.119 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:37.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:37.220 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:37.249 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:37.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:37.351 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:37.389 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:37.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:37.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:37.491 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:37.521 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:37.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:37.594 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:31:37.594 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:31:37.594 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:31:37.594 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:109911 10.2 10:41326339:::benchmark_data_smithi032_297285_object109910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:50.984185+0000", 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.606141555000001, 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.005982514, 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 109911 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.595 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:50.984185+0000", 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:50.984185+0000", 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:50.984187+0000", 2024-09-21T10:31:37.596 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.716e-06 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:50.984209+0000", 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2257000000000001e-05 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:50.984231+0000", 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1529999999999999e-05 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.597 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:50.984239+0000", 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.371e-06 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:50.985028+0000", 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00078907900000000004 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:50.985037+0000", 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6780000000000006e-06 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.990110+0000", 2024-09-21T10:31:37.598 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0050737489999999 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.990168+0000", 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.7133999999999999e-05 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:109953 10.2 10:440bd9ee:::benchmark_data_smithi032_297285_object109952:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.017605+0000", 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.572721235000003, 2024-09-21T10:31:37.599 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.98283363400000001, 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 109953 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.017605+0000", 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.600 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.017605+0000", 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.017607+0000", 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8160000000000001e-06 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.017632+0000", 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5109999999999998e-05 2024-09-21T10:31:37.601 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.017650+0000", 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7713000000000001e-05 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.017659+0000", 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6249999999999996e-06 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.017732+0000", 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3101000000000004e-05 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.602 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.017742+0000", 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0152e-05 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.990562+0000", 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.97281981799999995 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.990668+0000", 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000106548 2024-09-21T10:31:37.603 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.990771+0000", 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000102574 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000181+0000", 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0094098740000000004 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.604 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000241+0000", 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0704e-05 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000345+0000", 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000103181 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000399+0000", 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4685000000000002e-05 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.605 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000439+0000", 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9733000000000001e-05 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:109992 10.2 10:42d97849:::benchmark_data_smithi032_297285_object109991:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.060397+0000", 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.529929125000002, 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.94286167399999998, 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.606 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 109992 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.060397+0000", 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.607 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.608 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.608 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.060397+0000", 2024-09-21T10:31:37.608 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.608 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.608 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.608 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.608 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.060399+0000", 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.86e-06 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.060426+0000", 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.7083000000000001e-05 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.060444+0000", 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7727000000000002e-05 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.609 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.060455+0000", 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0448999999999999e-05 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.060940+0000", 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000485009 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.060964+0000", 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4468e-05 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.610 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.994022+0000", 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.93305812600000004 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.994175+0000", 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015237900000000001 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.994315+0000", 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000140922 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.611 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000506+0000", 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00619018 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000554+0000", 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7905999999999998e-05 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.003176+0000", 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0026229339999999999 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.612 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.003211+0000", 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4761999999999997e-05 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.003259+0000", 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7868999999999999e-05 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110036 10.2 10:4ee490e9:::benchmark_data_smithi032_297285_object110035:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.114991+0000", 2024-09-21T10:31:37.613 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.475335684999997, 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.89201378399999998, 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110036 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.114991+0000", 2024-09-21T10:31:37.614 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.114991+0000", 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.114993+0000", 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1679999999999998e-06 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.115018+0000", 2024-09-21T10:31:37.615 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4794e-05 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.115032+0000", 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4331999999999999e-05 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.115040+0000", 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0039999999999999e-06 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.115133+0000", 2024-09-21T10:31:37.616 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2763999999999995e-05 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.115145+0000", 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1885000000000001e-05 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.997107+0000", 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.88196216100000002 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.617 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.997253+0000", 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000146143 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.997403+0000", 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014986599999999999 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000605+0000", 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0032021549999999999 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.618 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.003270+0000", 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0026646090000000001 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.006820+0000", 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0035506240000000001 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.006937+0000", 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000116549 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.619 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.007005+0000", 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7730000000000004e-05 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110043 10.2 10:49b253d3:::benchmark_data_smithi032_297285_object110042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.122761+0000", 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.467566032000001, 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.88753179800000004, 2024-09-21T10:31:37.620 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110043 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.122761+0000", 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.621 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.122761+0000", 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.122762+0000", 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9039999999999999e-06 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.122786+0000", 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3896999999999998e-05 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.622 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.122805+0000", 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8668000000000002e-05 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.122815+0000", 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.645e-06 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.122889+0000", 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4636999999999997e-05 2024-09-21T10:31:37.623 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.122901+0000", 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2024999999999999e-05 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000649+0000", 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.87774727900000005 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000756+0000", 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000107644 2024-09-21T10:31:37.624 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.000845+0000", 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9085000000000005e-05 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.007060+0000", 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0062149839999999998 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.007106+0000", 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5265000000000001e-05 2024-09-21T10:31:37.625 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.010199+0000", 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0030931320000000002 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.010250+0000", 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1230000000000002e-05 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.010292+0000", 2024-09-21T10:31:37.626 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2403e-05 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110079 10.2 10:400a38fe:::benchmark_data_smithi032_297285_object110078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.167112+0000", 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.423214461000001, 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.84657285900000001, 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.627 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110079 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.167112+0000", 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.167112+0000", 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.628 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.167114+0000", 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8109999999999999e-06 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.167141+0000", 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6774000000000001e-05 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.167164+0000", 2024-09-21T10:31:37.629 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3210999999999999e-05 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.167174+0000", 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0448999999999999e-05 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.167261+0000", 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6558999999999996e-05 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.630 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.167283+0000", 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2486000000000001e-05 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.003344+0000", 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.83606049800000004 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.003491+0000", 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014670899999999999 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.631 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.003642+0000", 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015187600000000001 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.010299+0000", 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0066568010000000004 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.013545+0000", 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0032457290000000001 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.632 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.013589+0000", 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4345000000000003e-05 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.013633+0000", 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.392e-05 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.013685+0000", 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1690999999999997e-05 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.633 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110158 10.2 10:4ef37404:::benchmark_data_smithi032_297285_object110157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.253622+0000", 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.336704451000003, 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.76715302799999996, 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110158 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.634 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.253622+0000", 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.253622+0000", 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.253624+0000", 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.888e-06 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.635 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.253652+0000", 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.7858999999999999e-05 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.253670+0000", 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7943000000000001e-05 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.253679+0000", 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0480000000000004e-06 2024-09-21T10:31:37.636 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.253769+0000", 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0258000000000002e-05 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.253783+0000", 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3662000000000001e-05 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.007134+0000", 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.753350929 2024-09-21T10:31:37.637 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.007290+0000", 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000156687 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.007391+0000", 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000100237 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.013698+0000", 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0063074200000000002 2024-09-21T10:31:37.638 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.013773+0000", 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4474999999999994e-05 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.020469+0000", 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.006696011 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.020724+0000", 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00025548500000000001 2024-09-21T10:31:37.639 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.020775+0000", 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1125999999999998e-05 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110173 10.2 10:40bef63d:::benchmark_data_smithi032_297285_object110172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.271111+0000", 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.319215861000004, 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.74996821300000005, 2024-09-21T10:31:37.640 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110173 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.271111+0000", 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.641 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.271111+0000", 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.271112+0000", 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4729999999999999e-06 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.271133+0000", 2024-09-21T10:31:37.642 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0534999999999999e-05 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.271145+0000", 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2053e-05 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.271151+0000", 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8730000000000004e-06 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.643 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.271216+0000", 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5665000000000002e-05 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.271227+0000", 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1041e-05 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.010327+0000", 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.73909978099999996 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.644 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.010469+0000", 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014190399999999999 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.010610+0000", 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014129799999999999 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.017160+0000", 2024-09-21T10:31:37.645 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0065495060000000001 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.020431+0000", 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003271604 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.020881+0000", 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00044982399999999998 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.646 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.021038+0000", 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000156861 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.021079+0000", 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.0794999999999998e-05 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.647 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.648 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110179 10.2 10:4685df7e:::benchmark_data_smithi032_297285_object110178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.276180+0000", 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.314146403999999, 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.76127076000000005, 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110179 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.649 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.276180+0000", 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.276180+0000", 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.276182+0000", 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.04e-06 2024-09-21T10:31:37.650 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.276206+0000", 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3400999999999998e-05 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.276224+0000", 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8532000000000001e-05 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.276233+0000", 2024-09-21T10:31:37.651 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4249999999999998e-06 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.276327+0000", 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4629000000000005e-05 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.276352+0000", 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5249000000000002e-05 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.013809+0000", 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.73745668399999997 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.652 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.013966+0000", 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015642400000000001 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.014099+0000", 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013370299999999999 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.028991+0000", 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.014891847999999999 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.653 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.029103+0000", 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011212799999999999 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037313+0000", 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0082100030000000004 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037387+0000", 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3893999999999994e-05 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.654 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037451+0000", 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3800000000000006e-05 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110180 10.2 10:41a2464c:::benchmark_data_smithi032_297285_object110179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.278112+0000", 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.312215045999999, 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.75945992900000003, 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.655 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110180 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.278112+0000", 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.278112+0000", 2024-09-21T10:31:37.656 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.278113+0000", 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6980000000000001e-06 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.278137+0000", 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3628000000000001e-05 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.278149+0000", 2024-09-21T10:31:37.657 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2434e-05 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.278157+0000", 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7579999999999992e-06 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.278222+0000", 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4657000000000001e-05 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.658 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.278231+0000", 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6719999999999999e-06 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.017219+0000", 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.73898714300000001 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.017333+0000", 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114605 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.659 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.017453+0000", 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011989099999999999 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.029088+0000", 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011634768 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.029108+0000", 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0321000000000001e-05 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037492+0000", 2024-09-21T10:31:37.660 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0083835569999999998 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037539+0000", 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7301e-05 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037571+0000", 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2496000000000002e-05 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.661 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110188 10.2 10:4686b98b:::benchmark_data_smithi032_297285_object110187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.287323+0000", 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.303003423, 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.75035807700000001, 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110188 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.287323+0000", 2024-09-21T10:31:37.662 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.287323+0000", 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.287325+0000", 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.536e-06 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.663 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.287350+0000", 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5522000000000001e-05 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.287364+0000", 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4232e-05 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.287373+0000", 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9879999999999993e-06 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.664 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.287444+0000", 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.0637999999999997e-05 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.287453+0000", 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.3470000000000005e-06 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.021111+0000", 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.73365787199999999 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.665 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.021245+0000", 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013420700000000001 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.021389+0000", 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014327799999999999 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.029113+0000", 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0077241740000000003 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.666 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.029154+0000", 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1044000000000002e-05 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037609+0000", 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0084547289999999994 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037650+0000", 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1650000000000003e-05 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.667 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037681+0000", 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0859999999999999e-05 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110202 10.2 10:49f15b6b:::benchmark_data_smithi032_297285_object110201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.307984+0000", 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.282342593999999, 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.72981401099999998, 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.668 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110202 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.307984+0000", 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.669 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.307984+0000", 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.307986+0000", 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.823e-06 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.308012+0000", 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5712e-05 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.670 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.308030+0000", 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8094e-05 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.308040+0000", 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0042e-05 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.308114+0000", 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4683000000000003e-05 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.308127+0000", 2024-09-21T10:31:37.671 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3028e-05 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.025277+0000", 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.71714983099999996 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.025437+0000", 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015974300000000001 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.025588+0000", 2024-09-21T10:31:37.672 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000151232 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.029165+0000", 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0035765950000000001 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037258+0000", 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0080933870000000005 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037716+0000", 2024-09-21T10:31:37.673 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00045798600000000002 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037763+0000", 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6600000000000001e-05 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037798+0000", 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.5255e-05 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.674 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110226 10.2 10:476734ac:::benchmark_data_smithi032_297285_object110225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.346973+0000", 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.243353497999998, 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.70006216099999996, 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110226 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.675 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.346973+0000", 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.346973+0000", 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.346975+0000", 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.21e-06 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.676 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.347001+0000", 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.5406999999999999e-05 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.347016+0000", 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5281000000000002e-05 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.347023+0000", 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4669999999999998e-06 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.677 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.347105+0000", 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1409999999999995e-05 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.347117+0000", 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1780000000000001e-05 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.029192+0000", 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.682075667 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.678 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.029329+0000", 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013666000000000001 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.029494+0000", 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016516400000000001 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037282+0000", 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0077880299999999996 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.046628+0000", 2024-09-21T10:31:37.679 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0093459319999999995 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.046721+0000", 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2528000000000006e-05 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.046979+0000", 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00025839599999999998 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.047035+0000", 2024-09-21T10:31:37.680 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6229e-05 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110234 10.2 10:47f7616a:::benchmark_data_smithi032_297285_object110233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.365470+0000", 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.224856696000003, 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.68174660399999998, 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110234 2024-09-21T10:31:37.681 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.365470+0000", 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.365470+0000", 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.365472+0000", 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.688e-06 2024-09-21T10:31:37.682 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.365498+0000", 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6574e-05 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.365516+0000", 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7935e-05 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.365525+0000", 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.0440000000000001e-06 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.683 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.365594+0000", 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8672999999999998e-05 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.365606+0000", 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1833e-05 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.032746+0000", 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.667140022 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.684 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.032884+0000", 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013880500000000001 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.033849+0000", 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00096411000000000001 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037809+0000", 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0039608719999999998 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.685 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.046681+0000", 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0088716699999999999 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.047077+0000", 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000395929 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.047182+0000", 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000104938 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.686 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.047216+0000", 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.4511000000000002e-05 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110245 10.2 10:43eebb76:::benchmark_data_smithi032_297285_object110244:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.387929+0000", 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.202397546999997, 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.66834859999999996, 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.687 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110245 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.387929+0000", 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.387929+0000", 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.688 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.387931+0000", 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6050000000000001e-06 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.387980+0000", 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8897999999999998e-05 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.387995+0000", 2024-09-21T10:31:37.689 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5197999999999999e-05 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.388004+0000", 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9479999999999997e-06 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.388071+0000", 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7416999999999995e-05 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.690 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.388080+0000", 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2210000000000003e-06 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037834+0000", 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.64975375000000002 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.037962+0000", 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012816699999999999 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.691 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.038108+0000", 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014609900000000001 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.055735+0000", 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.017627035999999999 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.055751+0000", 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5753999999999999e-05 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.692 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.056055+0000", 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0003035 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.056232+0000", 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017734599999999999 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.056278+0000", 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5661000000000002e-05 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.693 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:110251 10.2 10:4cd6ddbf:::benchmark_data_smithi032_297285_object110250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:51.417094+0000", 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 46.173232902999999, 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.64004238999999996, 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 110251 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.694 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.417094+0000", 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.417094+0000", 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.417096+0000", 2024-09-21T10:31:37.695 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.08e-06 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.417140+0000", 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3893999999999997e-05 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.417153+0000", 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3108999999999999e-05 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.417159+0000", 2024-09-21T10:31:37.696 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.6320000000000002e-06 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.417218+0000", 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8462999999999999e-05 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:51.417225+0000", 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6450000000000005e-06 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.697 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.041779+0000", 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.62455351000000003 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.041959+0000", 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018000000000000001 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.042113+0000", 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000153571 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.698 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.699 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.055757+0000", 2024-09-21T10:31:37.699 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013644062 2024-09-21T10:31:37.699 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.699 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.699 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.055829+0000", 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2785000000000002e-05 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.057010+0000", 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0011802589999999999 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.057097+0000", 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7415000000000002e-05 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.700 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:52.057136+0000", 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8964999999999997e-05 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:122587 10.b e454/451)", 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:31:06.091598+0000", 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.498728731, 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.33062066600000001, 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.701 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.091598+0000", 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.091598+0000", 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.702 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.091599+0000", 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0780000000000001e-06 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.091607+0000", 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3960000000000006e-06 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.091620+0000", 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2916e-05 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.703 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.091630+0000", 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4080000000000004e-06 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.422194+0000", 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.33056445499999998 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.422219+0000", 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4413000000000002e-05 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.704 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:122593 10.2 10:4530e6f0:::benchmark_data_smithi032_297285_object5131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:31:06.126421+0000", 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.463905106999999, 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.002358761, 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 122593 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.705 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126421+0000", 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126421+0000", 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126423+0000", 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.933e-06 2024-09-21T10:31:37.706 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126427+0000", 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.117e-06 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126437+0000", 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0954e-05 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.707 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126445+0000", 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7700000000000001e-06 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.423218+0000", 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.29677288200000002 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.423224+0000", 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2049999999999999e-06 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.708 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.115921+0000", 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.69269715700000001 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.116007+0000", 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.5156999999999998e-05 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4", 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.116094+0000", 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7566999999999997e-05 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.122080+0000", 2024-09-21T10:31:37.709 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0059860130000000001 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.128625+0000", 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.006544561 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.128678+0000", 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3081999999999998e-05 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.710 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.128780+0000", 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000102363 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:122595 10.2 10:4adb7f42:::benchmark_data_smithi032_297285_object5133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:31:06.126458+0000", 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.463868495, 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.002467607, 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.711 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 122595 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126458+0000", 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126458+0000", 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.712 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126459+0000", 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2199999999999999e-07 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126461+0000", 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0660000000000002e-06 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126466+0000", 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9059999999999999e-06 2024-09-21T10:31:37.713 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.126473+0000", 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2149999999999998e-06 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.423252+0000", 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.29677906300000001 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.423256+0000", 2024-09-21T10:31:37.714 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9890000000000003e-06 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.118783+0000", 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.69552693099999996 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.118871+0000", 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8687e-05 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4", 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.118936+0000", 2024-09-21T10:31:37.715 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4623999999999997e-05 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.122089+0000", 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0031530009999999999 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.128821+0000", 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0067321799999999999 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.716 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.128853+0000", 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1655000000000001e-05 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.128926+0000", 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2867999999999994e-05 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:122596 10.2 10:4e9704c8:::benchmark_data_smithi032_297285_object5134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:31:06.438410+0000", 2024-09-21T10:31:37.717 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 31.151916427, 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.69579511599999999, 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 122596 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:31:37.718 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.438410+0000", 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.438410+0000", 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.438412+0000", 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6780000000000001e-06 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.719 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.438418+0000", 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1059999999999996e-06 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.438710+0000", 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00029194799999999999 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.438721+0000", 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1396999999999999e-05 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.720 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.438804+0000", 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3092999999999999e-05 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.438811+0000", 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4280000000000001e-06 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.120727+0000", 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.68191573699999997 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.721 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.120794+0000", 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7341000000000005e-05 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 4", 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.120859+0000", 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5228999999999995e-05 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.122207+0000", 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001347453 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.722 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.134052+0000", 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.01184583 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.134106+0000", 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3835999999999998e-05 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.134205+0000", 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.904e-05 2024-09-21T10:31:37.723 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.724 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.724 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.724 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:31:37.724 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:31:37.724 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:31:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:37.736 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:37.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:37.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:37.864 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:37.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:37.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:37.965 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:38.000 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:38.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:38.102 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:38.145 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:38.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:38.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:37 smithi032 ceph-mon[101299]: pgmap v1431: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 50991/244668 objects degraded (20.841%) 2024-09-21T10:31:38.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:37 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 50991/244668 objects degraded (20.841%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:38.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:37 smithi032 ceph-mon[105748]: pgmap v1431: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 50991/244668 objects degraded (20.841%) 2024-09-21T10:31:38.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:37 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 50991/244668 objects degraded (20.841%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:38.246 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:37 smithi045 ceph-mon[95976]: pgmap v1431: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 50991/244668 objects degraded (20.841%) 2024-09-21T10:31:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:37 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 50991/244668 objects degraded (20.841%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:38.285 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:38.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:38.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:38.387 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:38.433 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:38.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:38.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:38.535 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:38.579 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:38.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:38.680 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:38.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:38.719 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:38.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:38.820 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:38.845 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:38.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:38.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:38.947 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:38.976 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:38.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:38.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:39.078 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:39.106 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:39.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:39.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:39.208 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:39.240 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:39.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:39.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:39.342 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:39.388 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:39.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:39.490 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:39.517 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:39.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:39.619 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:39.670 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:39.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:39.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:39.772 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:39.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:39.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:39.863 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:31:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:31:39] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.51.0" 2024-09-21T10:31:39.902 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:39.946 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:39.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:39.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:40.048 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:40.092 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:40.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:40.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:40.132 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:39 smithi045 ceph-mon[95976]: pgmap v1432: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 50991/244668 objects degraded (20.841%) 2024-09-21T10:31:40.132 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:31:40.194 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:39 smithi032 ceph-mon[105748]: pgmap v1432: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 50991/244668 objects degraded (20.841%) 2024-09-21T10:31:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:31:40.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:39 smithi032 ceph-mon[101299]: pgmap v1432: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 50991/244668 objects degraded (20.841%) 2024-09-21T10:31:40.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:31:40.238 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:40.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:40.340 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:40.394 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:40.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:40.417 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:31:40.496 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:40.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:40.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:40.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:40.627 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:40.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:40.682 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:40.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:40.783 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:40.830 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:40.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:40.932 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:40.980 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:40.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:40.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:41.082 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:41.112 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:41.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:41.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:41.214 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:41.254 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:41.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:41.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:41.356 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:41.384 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:41.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:41.486 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:41.514 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:41.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:41.616 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:41.656 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:41.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:41.705 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:41.716 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:31:41.758 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:41.788 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:41.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:41.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:41.890 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:41.952 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:41.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:41.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:41.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:41 smithi032 ceph-mon[105748]: pgmap v1433: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 48188/231348 objects degraded (20.829%) 2024-09-21T10:31:41.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:41 smithi032 ceph-mon[101299]: pgmap v1433: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 48188/231348 objects degraded (20.829%) 2024-09-21T10:31:42.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:42.080 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:42.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:42.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:42.182 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:42.235 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:42.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:41 smithi045 ceph-mon[95976]: pgmap v1433: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 48188/231348 objects degraded (20.829%) 2024-09-21T10:31:42.336 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:42.368 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:42.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:42.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:42.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:42.527 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:42.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:42.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:42.628 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:42.657 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:42.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:42.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:42.758 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:42.791 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:42.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:42.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:42.893 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:42.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:42.949 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:42.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:42.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:42 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 48188/231348 objects degraded (20.829%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:42.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:42 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 48188/231348 objects degraded (20.829%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:43.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:42 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 48188/231348 objects degraded (20.829%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:43.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:43.077 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:43.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:43.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:43.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:43.208 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:43.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:43.310 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:43.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:43.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:43.440 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:43.481 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:43.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:43.583 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:43.622 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:31:43.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:43.623 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:43.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:43.657 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:31:43.657 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:31:43.660 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:43.691 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:43.699 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:31:43.700 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:31:43.723 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:43.736 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:31:43.736 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:31:43.759 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:43.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:43.775 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:31:43.775 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:31:43.800 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:31:43.800 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:31:43.831 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:31:43.832 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:31:43.857 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:31:43.857 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:31:43.861 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:43.896 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:43.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:43.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:43.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:44.033 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:44.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:44.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:44.134 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:44.177 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:44.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:44.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:43 smithi032 ceph-mon[105748]: pgmap v1434: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 47502/228228 objects degraded (20.813%) 2024-09-21T10:31:44.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:43 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/850789062' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:43 smithi032 ceph-mon[101299]: pgmap v1434: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 47502/228228 objects degraded (20.813%) 2024-09-21T10:31:44.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:43 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/850789062' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:44.279 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:43 smithi045 ceph-mon[95976]: pgmap v1434: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 47502/228228 objects degraded (20.813%) 2024-09-21T10:31:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:43 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/850789062' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:44.310 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:44.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:44.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:44.412 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:44.441 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:44.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:44.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:44.542 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:44.580 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:44.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:44.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:44.682 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:44.714 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:44.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:44.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:44.815 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:44.844 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:44.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:44.946 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:44.984 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:44.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:44.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:45.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:45.108 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:45.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:45.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:45.210 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:45.239 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:45.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:45.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:45.341 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:45.372 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:45.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:45.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:45.474 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:45.510 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:45.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:45.611 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:45.639 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:45.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:45.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:45.741 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:45.778 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:45.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:45.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:45.880 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:45.889 INFO:teuthology.orchestra.run.smithi032.stderr:set pool 1 pgp_num to 21 2024-09-21T10:31:45.910 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:45.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:45.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:46.011 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:46.019 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:31:46.019 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:31:46.044 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:46.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:46.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:46.145 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:46.171 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:46.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:46.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:46.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:45 smithi032 ceph-mon[105748]: pgmap v1435: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 45613/218736 objects degraded (20.853%) 2024-09-21T10:31:46.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:45 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1421252981' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-21T10:31:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:45 smithi032 ceph-mon[101299]: pgmap v1435: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 45613/218736 objects degraded (20.853%) 2024-09-21T10:31:46.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:45 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1421252981' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-21T10:31:46.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:45 smithi045 ceph-mon[95976]: pgmap v1435: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 45613/218736 objects degraded (20.853%) 2024-09-21T10:31:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:45 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1421252981' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-21T10:31:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:46.324 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:46.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:46.425 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:46.459 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:46.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:46.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:46.561 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:46.598 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:46.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:46.699 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:46.722 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:46.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:46.824 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:46.853 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:46.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:46.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:46.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:46.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:46.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:46.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:47.080 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:47.115 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:47.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:47.145 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:46 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1421252981' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2024-09-21T10:31:47.146 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:46 smithi032 ceph-mon[101299]: osdmap e460: 8 total, 7 up, 8 in 2024-09-21T10:31:47.146 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:46 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1421252981' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2024-09-21T10:31:47.146 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:46 smithi032 ceph-mon[105748]: osdmap e460: 8 total, 7 up, 8 in 2024-09-21T10:31:47.217 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:47.239 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:47.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:46 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1421252981' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2024-09-21T10:31:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:46 smithi045 ceph-mon[95976]: osdmap e460: 8 total, 7 up, 8 in 2024-09-21T10:31:47.341 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:47.378 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:47.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:47.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:47.479 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:47.503 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:47.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:47.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:47.605 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:47.634 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:47.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:47.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:47.736 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:47.758 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:47.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:47.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:47.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:47.892 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:47.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:47.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:47.994 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:48.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:31:48.028 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:48.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:48.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:48.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:48.156 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:47 smithi032 ceph-mon[105748]: pgmap v1437: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 44307/212814 objects degraded (20.820%) 2024-09-21T10:31:48.156 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:47 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 44307/212814 objects degraded (20.820%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:48.156 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:47 smithi032 ceph-mon[101299]: pgmap v1437: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 44307/212814 objects degraded (20.820%) 2024-09-21T10:31:48.157 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:47 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 44307/212814 objects degraded (20.820%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:48.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:48.182 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:48.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:48.252 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:47 smithi045 ceph-mon[95976]: pgmap v1437: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 44307/212814 objects degraded (20.820%) 2024-09-21T10:31:48.252 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:47 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 44307/212814 objects degraded (20.820%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:48.282 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:48.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:48.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:48.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:48.434 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:48.480 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:48.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:48.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:48.581 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:48.621 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:48.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:48.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:48.722 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:48.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:48.753 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:48.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:48.854 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:48.885 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:48.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:48.987 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:49.026 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:49.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:49.126 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:49.158 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:31:49.158 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:31:49.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:49.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:49.159 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:49.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:49.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:49.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:49.172 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:49.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:49.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:49.288 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:31:49.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:49.289 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:31:49.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:49.326 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:49.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:49.426 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:49.458 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:49.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:49.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:49.559 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:49.586 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:49.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:49.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:49.688 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:49.722 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:49.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:49.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:49.824 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:49.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:49.851 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:49.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:49.902 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:31:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:31:49] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-09-21T10:31:49.953 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:49.982 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:49.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:49.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:50.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:50.122 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:50.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:50.191 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:49 smithi045 ceph-mon[95976]: pgmap v1438: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 44307/212814 objects degraded (20.820%) 2024-09-21T10:31:50.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:49 smithi032 ceph-mon[101299]: pgmap v1438: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 44307/212814 objects degraded (20.820%) 2024-09-21T10:31:50.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:49 smithi032 ceph-mon[105748]: pgmap v1438: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 44307/212814 objects degraded (20.820%) 2024-09-21T10:31:50.224 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:50.276 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:50.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:50.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:50.377 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:50.416 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:50.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:50.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:50.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:50.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:50.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:50.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:50.665 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:50.717 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:50.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:50.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:50.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:50.843 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:50.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:50.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:50.913 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:31:50.945 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:50.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:50.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:50.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:51.011 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:31:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:50 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2951221621' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:51.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:50 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:51.089 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:51.133 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:51.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:50 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2951221621' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:51.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:50 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:50 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2951221621' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:51.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:50 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:31:51.235 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:51.269 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:51.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:51.370 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:51.397 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:51.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:51.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:51.499 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:51.522 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:51.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:51.624 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:51.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:51.664 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:51.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:51.765 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:51.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:51.806 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:51.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:51.858 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:51.907 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:51.930 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:51.959 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:51.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:52.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:52.083 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:51 smithi032 ceph-mon[105748]: pgmap v1439: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 41821/200637 objects degraded (20.844%) 2024-09-21T10:31:52.084 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:51 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:31:52.084 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:51 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:31:52.084 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:51 smithi032 ceph-mon[105748]: osdmap e461: 8 total, 7 up, 8 in 2024-09-21T10:31:52.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:51 smithi032 ceph-mon[101299]: pgmap v1439: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 41821/200637 objects degraded (20.844%) 2024-09-21T10:31:52.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:51 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:31:52.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:51 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:31:52.084 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:51 smithi032 ceph-mon[101299]: osdmap e461: 8 total, 7 up, 8 in 2024-09-21T10:31:52.105 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:52.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:52.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:52.174 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 0, 4, 1] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 6, 7, 5] 2024-09-21T10:31:52.175 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:31:52.175 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [2, 3, 5, 6, 7, 0, 4, 1] 2024-09-21T10:31:52.175 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd out 0 2024-09-21T10:31:52.208 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:52.248 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:52.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:51 smithi045 ceph-mon[95976]: pgmap v1439: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 41821/200637 objects degraded (20.844%) 2024-09-21T10:31:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:51 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:31:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:51 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:31:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:51 smithi045 ceph-mon[95976]: osdmap e461: 8 total, 7 up, 8 in 2024-09-21T10:31:52.352 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:52.398 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:52.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:52.500 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:52.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:52.534 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:52.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:52.635 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:52.662 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:52.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:52.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:52.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:52.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:52.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:52.892 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:52.919 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:52.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:53.021 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:53.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:52 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 41821/200637 objects degraded (20.844%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:53.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:53.060 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:53.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:53.089 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:52 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 41821/200637 objects degraded (20.844%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:53.089 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:52 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 41821/200637 objects degraded (20.844%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:53.160 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:53.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:53.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:53.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:53.303 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:53.344 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:53.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:53.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:53.445 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:53.483 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:53.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:53.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:53.585 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:53.619 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:53.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:53.720 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:53.757 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:53.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:53.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:53.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:31:53.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:53.891 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:53.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:53.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:53.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:54.028 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:54.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:54.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:54.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:54.157 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:54.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:54.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:54.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:53 smithi032 ceph-mon[105748]: pgmap v1441: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 41192/197736 objects degraded (20.832%) 2024-09-21T10:31:54.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:53 smithi032 ceph-mon[101299]: pgmap v1441: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 41192/197736 objects degraded (20.832%) 2024-09-21T10:31:54.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:53 smithi045 ceph-mon[95976]: pgmap v1441: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 41192/197736 objects degraded (20.832%) 2024-09-21T10:31:54.299 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:54.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:54.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:54.400 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:54.439 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:54.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:54.541 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:54.578 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:54.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:54.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:54.679 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:54.721 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:54.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:54.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:54.823 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:54.868 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:54.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:54.969 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:55.006 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:55.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:55.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:55.107 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:55.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:55.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:55.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:55.249 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:55.276 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:55.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:55.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:55.378 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:31:55.416 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:55.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:31:55.517 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:55.560 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:55.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:55.661 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:55.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:55.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:55.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:55.795 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:55.826 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:55.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:55.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:55.928 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:55.960 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:55.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:55.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:56.005 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:55 smithi032 ceph-mon[101299]: pgmap v1442: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 39244/188034 objects degraded (20.871%) 2024-09-21T10:31:56.005 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:55 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/409035116' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:56.005 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:55 smithi032 ceph-mon[105748]: pgmap v1442: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 39244/188034 objects degraded (20.871%) 2024-09-21T10:31:56.005 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:55 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/409035116' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:56.062 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:56.095 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:56.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:56.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:56.197 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:56.273 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:31:56.274 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:31:56.274 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:31:56.274 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:31:56.274 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:31:56.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:55 smithi045 ceph-mon[95976]: pgmap v1442: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 39244/188034 objects degraded (20.871%) 2024-09-21T10:31:56.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:55 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/409035116' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:31:56.305 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:56.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:56.407 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:56.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:56.434 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:56.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:56.535 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:56.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:56.570 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:56.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:56.611 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:31:56.611 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:31:56.611 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:31:56.611 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:31:56.612 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:31:56.612 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:31:56.612 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:31:56.612 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:31:56.612 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:31:56.612 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:31:56.612 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:31:56.612 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:31:56.613 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:31:56.613 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:31:56.613 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:31:56.671 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:56.700 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:56.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:56.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:56.802 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:31:56.838 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:56.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:56.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:31:56.939 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:56.981 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:56.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:56.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:57.083 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:57.113 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:57.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:57.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:57.215 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:57.257 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:57.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:57.359 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:57.389 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:31:57.390 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:31:57.394 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:57.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:57.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:57.405 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:57.405 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:31:57.496 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:57.530 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:57.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:57.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:57.632 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:57.662 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:57.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:57.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:57.764 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:57.798 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:57.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:57.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:57.899 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:57.931 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:57.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:57.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:57.956 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:31:58.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:58.087 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:58.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:58.132 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:57 smithi032 ceph-mon[101299]: pgmap v1443: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 37964/182157 objects degraded (20.841%) 2024-09-21T10:31:58.132 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:57 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 37964/182157 objects degraded (20.841%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:58.132 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:57 smithi032 ceph-mon[105748]: pgmap v1443: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 37964/182157 objects degraded (20.841%) 2024-09-21T10:31:58.132 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:57 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 37964/182157 objects degraded (20.841%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:58.188 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:58.237 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:58.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:58.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:57 smithi045 ceph-mon[95976]: pgmap v1443: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 37964/182157 objects degraded (20.841%) 2024-09-21T10:31:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:57 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 37964/182157 objects degraded (20.841%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:31:58.339 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:58.379 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:58.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:58.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:58.481 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:58.523 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:58.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:58.625 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:31:58.663 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:58.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:58.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:31:58.764 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:58.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:58.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:58.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:58.899 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:58.936 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:58.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:59.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:59.078 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:59.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:59.180 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:59.213 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:59.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:59.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:59.314 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:59.357 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:59.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:59.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:59.390 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:31:59.459 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:31:59.496 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:59.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:31:59.597 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:31:59.642 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:59.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:31:59.744 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:31:59.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:59.780 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:31:59.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:31:59.880 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:31:59.904 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:31:59.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:31:59.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:31:59.936 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:31:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:31:59] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-09-21T10:32:00.006 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:00.036 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:00.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:00.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:00.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:00.168 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:00.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:00.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:00.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:31:59 smithi032 ceph-mon[105748]: pgmap v1444: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 37964/182157 objects degraded (20.841%) 2024-09-21T10:32:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:31:59 smithi032 ceph-mon[101299]: pgmap v1444: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 37964/182157 objects degraded (20.841%) 2024-09-21T10:32:00.270 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:31:59 smithi045 ceph-mon[95976]: pgmap v1444: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 37964/182157 objects degraded (20.841%) 2024-09-21T10:32:00.306 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:00.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:00.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:00.408 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:00.448 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:00.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:00.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:00.550 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:00.582 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:00.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:00.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:00.684 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:00.717 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:00.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:00.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:00.819 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:00.855 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:00.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:00.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:00.993 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:00.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:00.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:01.094 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:01.129 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:01.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:01.231 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:01.274 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:01.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:01.376 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:01.404 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:01.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:01.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:01.505 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:01.549 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:01.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:01.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:01.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:01.687 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:01.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:01.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:01.788 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:01.822 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:01.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:01.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:01.923 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:01.956 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:01.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:02.058 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:02.094 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:02.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:02.195 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:02.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:01 smithi032 ceph-mon[101299]: pgmap v1445: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 35316/168990 objects degraded (20.898%) 2024-09-21T10:32:02.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:01 smithi032 ceph-mon[105748]: pgmap v1445: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 35316/168990 objects degraded (20.898%) 2024-09-21T10:32:02.222 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:02.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:02.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:01 smithi045 ceph-mon[95976]: pgmap v1445: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 35316/168990 objects degraded (20.898%) 2024-09-21T10:32:02.323 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:02.350 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:02.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:02.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:02.451 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:02.491 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:02.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:02.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:02.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:02.618 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:02.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:02.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:02.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:02.758 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:02.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:02.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:02.860 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:02.896 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:02.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:02.999 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:03.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:03.035 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:03.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:03.136 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:03.159 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:03.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:03.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:02 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 35316/168990 objects degraded (20.898%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:03.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:02 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 35316/168990 objects degraded (20.898%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:03.261 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:03.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:02 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 35316/168990 objects degraded (20.898%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:03.309 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:03.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:03.411 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:03.457 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:03.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:03.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:03.559 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:03.595 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:03.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:03.696 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:03.733 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:03.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:03.834 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:03.872 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:03.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:03.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:04.017 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:04.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:04.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:04.119 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:04.149 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:04.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:04.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:04.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:03 smithi032 ceph-mon[105748]: pgmap v1446: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 34669/165960 objects degraded (20.890%) 2024-09-21T10:32:04.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:04.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:04.209 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:03 smithi032 ceph-mon[101299]: pgmap v1446: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 34669/165960 objects degraded (20.890%) 2024-09-21T10:32:04.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:04.210 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:04.251 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:03 smithi045 ceph-mon[95976]: pgmap v1446: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 34669/165960 objects degraded (20.890%) 2024-09-21T10:32:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:04.290 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:04.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:04.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:04.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:04.441 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:04.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:04.542 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:04.567 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:04 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:32:04.215+0000 7f532c355640 -1 mon.a@0(leader).osd e461 definitely_dead 0 2024-09-21T10:32:04.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:04.578 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:04.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:04.678 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:04.719 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:04.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:04.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:04.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:04.854 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:04.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:04.955 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:04.956 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:32:04.957 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:04.958 INFO:teuthology.orchestra.run.smithi032.stderr:marked out osd.0. 2024-09-21T10:32:04.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:04.992 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:04.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:05.093 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:05.116 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:05.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:05.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:05.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:04 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1046209156' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:32:05.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:04 smithi032 ceph-mon[101299]: Client client.admin marked osd.0 out, after it was down for 58 seconds 2024-09-21T10:32:05.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:04 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/620677695' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:32:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:04 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1046209156' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:32:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:04 smithi032 ceph-mon[105748]: Client client.admin marked osd.0 out, after it was down for 58 seconds 2024-09-21T10:32:05.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:04 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/620677695' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:32:05.218 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:05.240 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:05.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:05.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:04 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1046209156' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:32:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:04 smithi045 ceph-mon[95976]: Client client.admin marked osd.0 out, after it was down for 58 seconds 2024-09-21T10:32:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:04 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/620677695' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:32:05.342 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:05.365 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:05.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:05.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:05.467 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:05.489 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:05.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:05.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:05.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:05.613 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:05.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:05.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:05.715 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:05.739 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:05.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:05.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:05.841 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:05.864 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:05.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:05.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:05.966 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:06.000 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:06.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:06.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:06.101 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:06.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:06.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:06.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[101299]: pgmap v1447: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 32588/155730 objects degraded (20.926%) 2024-09-21T10:32:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:32:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[101299]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:32:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1046209156' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-21T10:32:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/620677695' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:32:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[101299]: osdmap e462: 8 total, 7 up, 7 in 2024-09-21T10:32:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[105748]: pgmap v1447: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 32588/155730 objects degraded (20.926%) 2024-09-21T10:32:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:32:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[105748]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:32:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1046209156' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-21T10:32:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/620677695' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:32:06.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:05 smithi032 ceph-mon[105748]: osdmap e462: 8 total, 7 up, 7 in 2024-09-21T10:32:06.230 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:06.254 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:06.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:06.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:05 smithi045 ceph-mon[95976]: pgmap v1447: 109 pgs: 23 active+undersized, 19 active+undersized+degraded, 67 active+clean; 3.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 32588/155730 objects degraded (20.926%) 2024-09-21T10:32:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:05 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:32:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:05 smithi045 ceph-mon[95976]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:32:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1046209156' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-21T10:32:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:05 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/620677695' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:32:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:05 smithi045 ceph-mon[95976]: osdmap e462: 8 total, 7 up, 7 in 2024-09-21T10:32:06.355 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:06.378 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:06.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:06.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:06.480 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:06.507 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:06.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:06.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:06.608 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:06.634 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:06.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:06.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:06.736 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:06.762 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:06.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:06.864 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:06.887 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:06.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:06.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:06.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:07.027 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:07.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:07.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:07.129 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:07.165 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:07.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:07.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:07.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:06 smithi032 ceph-mon[101299]: osdmap e463: 8 total, 7 up, 7 in 2024-09-21T10:32:07.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:06 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 32588/155730 objects degraded (20.926%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:07.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:06 smithi032 ceph-mon[105748]: osdmap e463: 8 total, 7 up, 7 in 2024-09-21T10:32:07.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:06 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 32588/155730 objects degraded (20.926%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:07.267 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:06 smithi045 ceph-mon[95976]: osdmap e463: 8 total, 7 up, 7 in 2024-09-21T10:32:07.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:06 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 32588/155730 objects degraded (20.926%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:07.301 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:07.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:07.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:07.403 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:07.449 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:07.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:07.551 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:07.590 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:07.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:07.691 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:07.722 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:07.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:07.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:07.823 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:07.866 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:07.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:07.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:07.968 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:08.026 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:08.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:08.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:08.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:08.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:08.164 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:08.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: pgmap v1450: 109 pgs: 6 remapped+peering, 15 active+undersized, 14 active+undersized+degraded, 74 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s; 25142/150384 objects degraded (16.719%); 72 B/s, 0 objects/s recovering 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.2 starting backfill to osd.3 from (0'0,0'0] MAX to 461'11658 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.6 starting backfill to osd.1 from (0'0,0'0] MAX to 461'11557 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: osdmap e464: 8 total, 7 up, 7 in 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 461'11607 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 461'11561 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.4 starting backfill to osd.1 from (0'0,0'0] MAX to 461'11677 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 461'11835 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.5 starting backfill to osd.7 from (0'0,0'0] MAX to 461'11384 2024-09-21T10:32:08.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 461'11408 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 461'11382 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[101299]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 461'11617 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: pgmap v1450: 109 pgs: 6 remapped+peering, 15 active+undersized, 14 active+undersized+degraded, 74 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s; 25142/150384 objects degraded (16.719%); 72 B/s, 0 objects/s recovering 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.2 starting backfill to osd.3 from (0'0,0'0] MAX to 461'11658 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.6 starting backfill to osd.1 from (0'0,0'0] MAX to 461'11557 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: osdmap e464: 8 total, 7 up, 7 in 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 461'11607 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 461'11561 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.4 starting backfill to osd.1 from (0'0,0'0] MAX to 461'11677 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 461'11835 2024-09-21T10:32:08.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.5 starting backfill to osd.7 from (0'0,0'0] MAX to 461'11384 2024-09-21T10:32:08.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 461'11408 2024-09-21T10:32:08.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 461'11382 2024-09-21T10:32:08.209 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:07 smithi032 ceph-mon[105748]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 461'11617 2024-09-21T10:32:08.265 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: pgmap v1450: 109 pgs: 6 remapped+peering, 15 active+undersized, 14 active+undersized+degraded, 74 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s; 25142/150384 objects degraded (16.719%); 72 B/s, 0 objects/s recovering 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.2 starting backfill to osd.3 from (0'0,0'0] MAX to 461'11658 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.6 starting backfill to osd.1 from (0'0,0'0] MAX to 461'11557 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: osdmap e464: 8 total, 7 up, 7 in 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 461'11607 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 461'11561 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.4 starting backfill to osd.1 from (0'0,0'0] MAX to 461'11677 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 461'11835 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.5 starting backfill to osd.7 from (0'0,0'0] MAX to 461'11384 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 461'11408 2024-09-21T10:32:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 461'11382 2024-09-21T10:32:08.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:07 smithi045 ceph-mon[95976]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 461'11617 2024-09-21T10:32:08.290 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:08.303 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:08.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:08.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:08.404 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:08.451 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:08.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:08.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:08.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:08.594 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:08.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:08.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:08.696 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:08.739 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:08.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:08.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:08.841 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:08.876 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:08.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:08.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:08.978 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:09.017 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:09.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:09.119 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:09.153 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:09.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:09.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:09.264 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:08 smithi032 ceph-mon[101299]: osdmap e465: 8 total, 7 up, 7 in 2024-09-21T10:32:09.264 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:08 smithi032 ceph-mon[105748]: osdmap e465: 8 total, 7 up, 7 in 2024-09-21T10:32:09.287 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:08 smithi045 ceph-mon[95976]: osdmap e465: 8 total, 7 up, 7 in 2024-09-21T10:32:09.294 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:09.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:09.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:09.395 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:09.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:09.438 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:09.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:09.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:09.572 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:09.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:09.674 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:09.712 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:09.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:09.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:09.814 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:09.861 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:09.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:09.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:09.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:32:09] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-09-21T10:32:09.963 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:10.008 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:10.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:10.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:10.151 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:10.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:10.253 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:09 smithi045 ceph-mon[95976]: pgmap v1453: 109 pgs: 6 remapped+peering, 15 active+undersized, 14 active+undersized+degraded, 74 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 25142/150384 objects degraded (16.719%); 145 B/s, 0 objects/s recovering 2024-09-21T10:32:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:09 smithi045 ceph-mon[95976]: osdmap e466: 8 total, 7 up, 7 in 2024-09-21T10:32:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:32:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:32:10.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:09 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:10.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:10.296 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:10.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:10.324 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:32:10.331 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[105748]: pgmap v1453: 109 pgs: 6 remapped+peering, 15 active+undersized, 14 active+undersized+degraded, 74 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 25142/150384 objects degraded (16.719%); 145 B/s, 0 objects/s recovering 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[105748]: osdmap e466: 8 total, 7 up, 7 in 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[101299]: pgmap v1453: 109 pgs: 6 remapped+peering, 15 active+undersized, 14 active+undersized+degraded, 74 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 25142/150384 objects degraded (16.719%); 145 B/s, 0 objects/s recovering 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[101299]: osdmap e466: 8 total, 7 up, 7 in 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:10.332 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:32:10.333 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:32:10.333 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:09 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:10.397 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:10.447 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:10.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:10.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:10.548 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:10.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:10.595 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:10.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:10.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:10.731 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:10.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:10.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:10.833 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:10.872 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:10.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:10.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:10.974 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:11.020 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:11.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:11.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:11.122 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:11.160 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:11.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:11.262 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:11.296 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:11.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:11.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:11.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:11.440 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:11.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:11.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:11.515 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4, 1] out_osds: [0] dead_osds: [0] live_osds: [1, 2, 3, 4, 6, 7, 5] 2024-09-21T10:32:11.516 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:32:11.516 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-21T10:32:11.516 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:32:11.516 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:32:11.516 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-21T10:32:11.542 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:11.585 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:11.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:11.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:11.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:11.719 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:11.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:11.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:11.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:11.855 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:11.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:11.926 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:32:11.957 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:11.967 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:32:11.967 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:32:11.993 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:11.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:11.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:12.010 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:32:12.010 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:32:12.010 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:32:12.010 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:32:12.010 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.010 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:172063 8.e e359/330 8:706f6b4b:::benchmark_data_smithi032_233595_object72218:head v 359'10968, mlcod=359'10965)", 2024-09-21T10:32:12.010 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:24.060777+0000", 2024-09-21T10:32:12.010 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 527.76843027699999, 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16745534200000001, 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.060777+0000", 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.060777+0000", 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.011 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.060778+0000", 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.75e-07 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.060782+0000", 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9339999999999999e-06 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.060789+0000", 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.2960000000000001e-06 2024-09-21T10:32:12.012 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.060794+0000", 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9609999999999996e-06 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.060836+0000", 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1692e-05 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.060855+0000", 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9021e-05 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.013 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228175+0000", 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.167319581 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228233+0000", 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.8282000000000002e-05 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.014 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:172073 8.e e359/330 8:78d536ff:::benchmark_data_smithi032_233595_object72228:head v 359'10969, mlcod=359'10966)", 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:24.063676+0000", 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 527.76553223500002, 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.164586858, 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.063676+0000", 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.015 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.063676+0000", 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.063676+0000", 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7100000000000001e-07 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.063682+0000", 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9959999999999999e-06 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.016 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.063693+0000", 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1153e-05 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.063700+0000", 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5899999999999996e-06 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.064479+0000", 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00077938599999999997 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.017 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.064498+0000", 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8987999999999998e-05 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228236+0000", 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16373796600000001 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228262+0000", 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6007999999999999e-05 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.018 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:172074 8.1 e359/305 8:83f7c06e:::benchmark_data_smithi032_233595_object72229:head v 359'10574, mlcod=359'10570)", 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:24.068790+0000", 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 527.76041745099997, 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.167604789, 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.068790+0000", 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.019 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.068790+0000", 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.068791+0000", 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5199999999999996e-07 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.068796+0000", 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5130000000000004e-06 2024-09-21T10:32:12.020 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.068807+0000", 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0156000000000001e-05 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.068814+0000", 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.951e-06 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.068893+0000", 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9331000000000003e-05 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.021 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.227974+0000", 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.159081373 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.236345+0000", 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0083712079999999998 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.236395+0000", 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9604999999999997e-05 2024-09-21T10:32:12.022 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:172078 8.f e359/305 8:fa10a93a:::benchmark_data_smithi032_233595_object72233:head v 359'10682, mlcod=359'10680)", 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:24.070789+0000", 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 527.75841895799999, 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.17380205800000001, 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.023 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.070789+0000", 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.070789+0000", 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.070790+0000", 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1200000000000001e-07 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.024 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.070797+0000", 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8580000000000002e-06 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228285+0000", 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.15748810099999999 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228292+0000", 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7279999999999995e-06 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.025 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228368+0000", 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5538999999999996e-05 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228385+0000", 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7017999999999999e-05 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.244545+0000", 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.016159738999999999 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.026 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.244591+0000", 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6162999999999998e-05 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:172072 8.d e359/339 8:b5cff8b6:::benchmark_data_smithi032_233595_object72227:head v 359'10689, mlcod=337'7500)", 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:24.075652+0000", 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 527.75355600399996, 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16287805, 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.027 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.075652+0000", 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.075652+0000", 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.075653+0000", 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1289999999999999e-06 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.028 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.075660+0000", 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1799999999999999e-06 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228554+0000", 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.15289389 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228564+0000", 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.6940000000000007e-06 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.029 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228606+0000", 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.286e-05 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.228649+0000", 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2987e-05 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.238447+0000", 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0097979279999999992 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.030 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:24.238530+0000", 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.2381999999999998e-05 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:192444 8.1 e367/305 8:829660cb:::benchmark_data_smithi032_233595_object92599:head v 367'11756, mlcod=367'11753)", 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:50.346119+0000", 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 501.48308862900001, 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.187267197, 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.031 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.346119+0000", 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.346119+0000", 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.346120+0000", 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0750000000000001e-06 2024-09-21T10:32:12.032 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.346127+0000", 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4559999999999997e-06 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.346138+0000", 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1657e-05 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.346145+0000", 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7710000000000001e-06 2024-09-21T10:32:12.033 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.346218+0000", 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3286000000000003e-05 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.346237+0000", 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8224e-05 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.349055+0000", 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0028182329999999998 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.034 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.533386+0000", 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.18433149500000001 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:192447 8.e e367/330 8:7f3a24ce:::benchmark_data_smithi032_233595_object92602:head v 367'12233, mlcod=367'12231)", 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:50.350829+0000", 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 501.47837921399997, 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.18961272200000001, 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.035 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.350829+0000", 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.350829+0000", 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.350829+0000", 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8999999999999995e-07 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.036 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.350836+0000", 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8650000000000003e-06 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.533423+0000", 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.18258675199999999 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.533431+0000", 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6249999999999998e-06 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.037 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.533471+0000", 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9981000000000002e-05 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.533490+0000", 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9504000000000002e-05 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.540353+0000", 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0068631600000000001 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.540441+0000", 2024-09-21T10:32:12.038 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7944999999999999e-05 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.35586.0:192435 8.4 e367/325 8:25d6fb97:::benchmark_data_smithi032_233595_object92590:head v 367'12060, mlcod=314'1920)", 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:23:50.353879+0000", 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 501.47532888900002, 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.20228510099999999, 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.039 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.353879+0000", 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.353879+0000", 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.353881+0000", 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8080000000000001e-06 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.040 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.353890+0000", 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1810000000000007e-06 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.533327+0000", 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.17943689800000001 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.533344+0000", 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7314000000000001e-05 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.041 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.533405+0000", 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1307000000000001e-05 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.533432+0000", 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.7101e-05 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.556119+0000", 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.022686976000000001 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.042 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:23:50.556164+0000", 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4515999999999999e-05 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:26472 10.5 10:a21b5266:::benchmark_data_smithi032_297285_object26471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:29:40.443219+0000", 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 151.38598891000001, 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.17561874399999999, 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.043 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26472 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443219+0000", 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443219+0000", 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.044 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443220+0000", 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6470000000000001e-06 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443243+0000", 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2765999999999999e-05 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443260+0000", 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.694e-05 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443268+0000", 2024-09-21T10:32:12.045 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1729999999999995e-06 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443447+0000", 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000179063 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443600+0000", 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000152299 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443736+0000", 2024-09-21T10:32:12.046 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000136155 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.446089+0000", 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0023527890000000001 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.449083+0000", 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002994493 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.618740+0000", 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16965655199999999 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.047 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.618796+0000", 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.6314999999999999e-05 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.618838+0000", 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1551999999999998e-05 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:26473 10.5 10:ad7b101a:::benchmark_data_smithi032_297285_object26472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:29:40.443293+0000", 2024-09-21T10:32:12.048 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 151.38591478000001, 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.175711742, 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26473 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443293+0000", 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.049 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443293+0000", 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443294+0000", 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1999999999999999e-07 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443321+0000", 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6922000000000001e-05 2024-09-21T10:32:12.050 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443333+0000", 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2293000000000001e-05 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.443365+0000", 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.2267000000000001e-05 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.444835+0000", 2024-09-21T10:32:12.051 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014694739999999999 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.444990+0000", 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015517499999999999 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.445126+0000", 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013590100000000001 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.446617+0000", 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0014909890000000001 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.052 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.449161+0000", 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.002544152 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.618901+0000", 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16974048999999999 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.618962+0000", 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1011000000000003e-05 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.053 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.619005+0000", 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2447999999999997e-05 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:26482 10.5 10:a63cada6:::benchmark_data_smithi032_297285_object26481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:29:40.447129+0000", 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 151.38207879500001, 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.17214455300000001, 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.054 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26482 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447129+0000", 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447129+0000", 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.055 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447130+0000", 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.277e-06 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447148+0000", 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.806e-05 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447160+0000", 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1919999999999999e-05 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.056 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447166+0000", 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4009999999999996e-06 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447215+0000", 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.9706999999999997e-05 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447506+0000", 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000290654 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447656+0000", 2024-09-21T10:32:12.057 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000149691 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.451042+0000", 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0033865649999999998 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.451056+0000", 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4045e-05 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.619179+0000", 2024-09-21T10:32:12.058 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16812292400000001 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.619234+0000", 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5090999999999998e-05 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.619273+0000", 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9218000000000003e-05 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.059 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:26483 10.5 10:a26189f6:::benchmark_data_smithi032_297285_object26482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:29:40.447778+0000", 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 151.381429457, 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.17164328300000001, 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26483 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.060 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447778+0000", 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447778+0000", 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447779+0000", 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.71e-07 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447803+0000", 2024-09-21T10:32:12.061 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.374e-05 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447812+0000", 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.5640000000000002e-06 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.447820+0000", 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1960000000000002e-06 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.062 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.449655+0000", 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0018357320000000001 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.449836+0000", 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00018013799999999999 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.449938+0000", 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000102668 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.063 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.451183+0000", 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0012449360000000001 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.451569+0000", 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000385807 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.619322+0000", 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16775290000000001 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.619386+0000", 2024-09-21T10:32:12.064 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3707999999999994e-05 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.619422+0000", 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6022999999999997e-05 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:26499 10.3 10:c94dae79:::benchmark_data_smithi032_297285_object26498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:29:40.454296+0000", 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 151.37491127499999, 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.1630142, 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.065 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 26499 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.454296+0000", 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.454296+0000", 2024-09-21T10:32:12.066 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.454298+0000", 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1909999999999999e-06 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.454315+0000", 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7487999999999999e-05 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.067 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.454325+0000", 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0268000000000001e-05 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.454331+0000", 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.4110000000000004e-06 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.454391+0000", 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.9716999999999998e-05 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.454492+0000", 2024-09-21T10:32:12.068 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000100997 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.454573+0000", 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1113999999999996e-05 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.455536+0000", 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00096289100000000001 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.456158+0000", 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00062240699999999995 2024-09-21T10:32:12.069 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.617216+0000", 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.16105788400000001 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.617260+0000", 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4209999999999999e-05 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:40.617311+0000", 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0621999999999997e-05 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.070 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:27605 10.f e424/419 10:f7e7a950:::benchmark_data_smithi032_297285_object27604:head v 424'1710, mlcod=424'1708)", 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:29:41.106608+0000", 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 150.72259978599999, 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27376502899999999, 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.106608+0000", 2024-09-21T10:32:12.071 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.106608+0000", 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.106609+0000", 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0999999999999997e-07 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.106657+0000", 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8208999999999999e-05 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.072 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.106665+0000", 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4840000000000004e-06 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.106670+0000", 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.843e-06 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.106710+0000", 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9289999999999998e-05 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.073 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.106722+0000", 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2497000000000001e-05 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.107769+0000", 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.001047166 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380373+0000", 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27260372999999999 2024-09-21T10:32:12.074 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:27618 10.3 10:c54f8f59:::benchmark_data_smithi032_297285_object27617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:29:41.109619+0000", 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 150.71958896199999, 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27662662900000001, 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 27618 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.075 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.109619+0000", 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.109619+0000", 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.109621+0000", 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.77e-06 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.076 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.109652+0000", 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.1717000000000001e-05 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380356+0000", 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27070328100000002 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380372+0000", 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6118000000000001e-05 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.077 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380433+0000", 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1209000000000003e-05 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380584+0000", 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000151469 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380727+0000", 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000142934 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.078 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.384583+0000", 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0038556689999999999 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.384621+0000", 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8355000000000001e-05 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.386134+0000", 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0015127280000000001 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.079 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.386195+0000", 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0822000000000001e-05 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.386245+0000", 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0556999999999997e-05 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:27615 10.9 e424/419 10:997d3ef7:::benchmark_data_smithi032_297285_object27614:head v 424'1813, mlcod=424'1811)", 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:29:41.110479+0000", 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 150.71872906900001, 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.272603555, 2024-09-21T10:32:12.080 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.110479+0000", 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.110479+0000", 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.081 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.110480+0000", 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.9700000000000005e-07 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.110541+0000", 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0940000000000003e-05 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380355+0000", 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.26981456799999998 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.082 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380371+0000", 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6390999999999999e-05 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380429+0000", 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.787e-05 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.380449+0000", 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9368000000000002e-05 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.383030+0000", 2024-09-21T10:32:12.083 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0025817409999999998 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:29:41.383082+0000", 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1780000000000002e-05 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:66967 10.3 10:cf1709e5:::benchmark_data_smithi032_297285_object66966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:06.091235+0000", 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 125.737972707, 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.211594956, 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.084 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 66967 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.091235+0000", 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.091235+0000", 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.085 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.091236+0000", 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1990000000000001e-06 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.091279+0000", 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.2833000000000001e-05 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.091294+0000", 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.5367999999999999e-05 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.086 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.091303+0000", 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.6410000000000008e-06 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.091480+0000", 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017737499999999999 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.091713+0000", 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00023269799999999999 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.087 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.091990+0000", 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00027658499999999998 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.098625+0000", 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0066357309999999997 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.099264+0000", 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00063889399999999999 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.302709+0000", 2024-09-21T10:32:12.088 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.20344494299999999 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.302779+0000", 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.9491000000000003e-05 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.302830+0000", 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.1198e-05 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.089 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:66989 10.3 10:cbd2f3cb:::benchmark_data_smithi032_297285_object66988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:06.109551+0000", 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 125.719656602, 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.20094796400000001, 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 66989 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.090 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.109551+0000", 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.109551+0000", 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.109553+0000", 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9250000000000002e-06 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.091 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.109574+0000", 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0441e-05 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.109593+0000", 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8998999999999999e-05 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.109600+0000", 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8439999999999998e-06 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.092 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.109654+0000", 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3152999999999997e-05 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.109823+0000", 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000169987 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.109972+0000", 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014862999999999999 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.112506+0000", 2024-09-21T10:32:12.093 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0025339270000000001 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.113410+0000", 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00090361800000000002 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.310387+0000", 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.19697742900000001 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.310455+0000", 2024-09-21T10:32:12.094 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7527999999999994e-05 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.310499+0000", 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4483000000000001e-05 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:67005 10.3 10:c81f6cbe:::benchmark_data_smithi032_297285_object67004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:30:06.123617+0000", 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 125.70559121700001, 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.19273960100000001, 2024-09-21T10:32:12.095 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 67005 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.123617+0000", 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.096 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.123617+0000", 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.123618+0000", 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.584e-06 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.123637+0000", 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8916e-05 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.097 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.123652+0000", 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4778e-05 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.123658+0000", 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.2790000000000004e-06 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.123706+0000", 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7513999999999999e-05 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.098 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.123825+0000", 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00011892000000000001 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.123931+0000", 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00010604500000000001 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.125928+0000", 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0019975069999999999 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.099 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.126150+0000", 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00022236000000000001 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.316224+0000", 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.19007384899999999 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.316300+0000", 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5957000000000005e-05 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:30:06.316356+0000", 2024-09-21T10:32:12.100 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5892e-05 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.36234.0:122575 10.5 10:a0bfb4d4:::benchmark_data_smithi032_297285_object5113:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:31:06.069429+0000", 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 65.759778994000001, 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0417690079999999, 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "reached pg", 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.36234", 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:32:12.101 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 122575 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.069429+0000", 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.069429+0000", 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.069430+0000", 2024-09-21T10:32:12.102 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.341e-06 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.069434+0000", 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.313e-06 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.069445+0000", 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0547000000000001e-05 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.069455+0000", 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0281e-05 2024-09-21T10:32:12.103 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.083207+0000", 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.013751578 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.083365+0000", 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015784499999999999 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.083506+0000", 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014181999999999999 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.104 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.089305+0000", 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0057980460000000003 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.105044+0000", 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.015739013 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for peered", 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:06.105055+0000", 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1619e-05 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.111116+0000", 2024-09-21T10:32:12.105 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0060610000000001 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:31:07.111198+0000", 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1605000000000002e-05 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:12.106 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:32:12.106 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:11 smithi032 ceph-mon[105748]: pgmap v1455: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 6 remapped+peering, 95 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 22964/143838 objects degraded (15.965%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:32:12.107 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:11 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4087172239' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:12.107 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:11 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 22964/143838 objects degraded (15.965%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:12.107 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:11 smithi032 ceph-mon[101299]: pgmap v1455: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 6 remapped+peering, 95 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 22964/143838 objects degraded (15.965%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:32:12.107 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:11 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4087172239' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:12.107 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:11 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 22964/143838 objects degraded (15.965%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:12.107 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:32:12.107 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:32:12.108 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:32:12.108 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:32:12.108 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:32:12.109 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:32:12.109 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:32:12.109 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:32:12.109 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:12.135 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:32:12.135 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:32:12.147 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:12.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:12.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:12.169 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:32:12.170 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:32:12.248 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:12.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:12.285 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:12.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:11 smithi045 ceph-mon[95976]: pgmap v1455: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 6 remapped+peering, 95 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 22964/143838 objects degraded (15.965%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:32:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:11 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4087172239' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:11 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 22964/143838 objects degraded (15.965%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:12.386 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:12.423 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:12.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:12.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:12.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:12.565 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:12.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:12.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:12.666 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:12.703 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:12.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:12.804 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:12.845 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:12.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:12.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:12.946 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:12.984 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:12.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:12.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:13.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:13.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:13.120 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:13.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:13.221 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:13.256 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:13.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:13.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:13.358 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:13.390 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:13.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:13.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:13.491 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:13.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:13.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:13.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:13.627 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:13.666 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:13.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:13.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:13.768 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:13.808 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:13.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:13.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:13.909 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:13.911 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:32:13.990 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:13.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:14.092 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:14.133 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:14.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:14.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:14.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:13 smithi032 ceph-mon[101299]: pgmap v1456: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 27645/141612 objects degraded (19.522%); 21 MiB/s, 339 objects/s recovering 2024-09-21T10:32:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:13 smithi032 ceph-mon[105748]: pgmap v1456: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 27645/141612 objects degraded (19.522%); 21 MiB/s, 339 objects/s recovering 2024-09-21T10:32:14.234 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:14.272 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:14.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:13 smithi045 ceph-mon[95976]: pgmap v1456: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 27645/141612 objects degraded (19.522%); 21 MiB/s, 339 objects/s recovering 2024-09-21T10:32:14.374 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:14.416 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:14.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:14.454 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:14.484 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:32:14.484 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:32:14.518 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:14.531 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:14.565 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:14.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:14.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:14.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:14.713 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:14.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:14.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:14.815 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:14.858 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:14.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:14.959 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:14.998 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:14.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:15.100 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:15.144 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:15.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:15.246 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:15.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:15.288 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:15.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:15.389 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:15.428 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:15.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:15.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:15.529 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:15.569 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:15.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:15.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:15.670 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:15.710 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:15.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:15.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:15.811 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:15.847 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:15.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:15.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:15.949 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:15.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:15.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:15.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:16.088 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:16.116 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:16.125 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:16.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:16.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:16.227 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:16.267 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:16 smithi032 ceph-mon[105748]: pgmap v1457: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 24634/132456 objects degraded (18.598%); 27 MiB/s, 428 objects/s recovering 2024-09-21T10:32:16.268 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:16 smithi032 ceph-mon[101299]: pgmap v1457: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 24634/132456 objects degraded (18.598%); 27 MiB/s, 428 objects/s recovering 2024-09-21T10:32:16.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:16 smithi045 ceph-mon[95976]: pgmap v1457: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 24634/132456 objects degraded (18.598%); 27 MiB/s, 428 objects/s recovering 2024-09-21T10:32:16.298 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:16.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:16.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:16.399 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:16.437 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:16.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:16.485 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:32:16.539 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:16.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:16.580 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:16.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:16.680 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:16.720 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:16.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:16.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:16.822 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:16.852 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:16.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:16.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:16.954 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:16.999 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:17.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:17.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:17.101 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:17.139 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:17.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:17.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:17.241 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:17.276 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:17.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:17.280 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:17 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 24634/132456 objects degraded (18.598%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:17.280 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:17 smithi032 ceph-mon[101299]: osdmap e467: 8 total, 7 up, 7 in 2024-09-21T10:32:17.280 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:17 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 24634/132456 objects degraded (18.598%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:17.281 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:17 smithi032 ceph-mon[105748]: osdmap e467: 8 total, 7 up, 7 in 2024-09-21T10:32:17.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:17 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 24634/132456 objects degraded (18.598%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:17.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:17 smithi045 ceph-mon[95976]: osdmap e467: 8 total, 7 up, 7 in 2024-09-21T10:32:17.377 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:17.407 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:17.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:17.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:17.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:17.538 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:17.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:17.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:17.640 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:17.669 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:17.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:17.771 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:17.801 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:17.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:17.903 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:17.932 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:17.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:17.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:18.034 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:18.067 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:18.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:18.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:18.168 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:18.205 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:18.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:18.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:18 smithi045 ceph-mon[95976]: pgmap v1458: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 21679/127203 objects degraded (17.043%); 39 MiB/s, 613 objects/s recovering 2024-09-21T10:32:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:18 smithi045 ceph-mon[95976]: osdmap e468: 8 total, 7 up, 7 in 2024-09-21T10:32:18.307 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:18.338 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:18.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:18.410 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:18 smithi032 ceph-mon[101299]: pgmap v1458: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 21679/127203 objects degraded (17.043%); 39 MiB/s, 613 objects/s recovering 2024-09-21T10:32:18.410 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:18 smithi032 ceph-mon[101299]: osdmap e468: 8 total, 7 up, 7 in 2024-09-21T10:32:18.410 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:18 smithi032 ceph-mon[105748]: pgmap v1458: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 21679/127203 objects degraded (17.043%); 39 MiB/s, 613 objects/s recovering 2024-09-21T10:32:18.410 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:18 smithi032 ceph-mon[105748]: osdmap e468: 8 total, 7 up, 7 in 2024-09-21T10:32:18.440 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:18.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:18.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:18.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:18.578 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:18.608 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:18.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:18.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:18.710 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:18.756 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:18.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:18.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:18.858 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:18.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:18.915 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:18.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:19.016 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:19.036 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:32:19.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:19.065 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:19.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:19.166 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/715770725' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:32:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/715770725' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:32:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:19.208 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:19.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:19.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/715770725' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:32:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:19.311 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:19.349 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:19.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:19.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:19.451 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:19.494 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:19.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:19.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:19.596 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:19.630 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:19.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:19.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:19.696 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:32:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:32:19] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.51.0" 2024-09-21T10:32:19.732 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:19.772 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:19.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:19.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:19.874 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:19.910 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:19.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:20.011 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:20.052 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:20.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:20.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:20.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:20.192 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:20.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:20.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:20.215 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:20.245 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:32:20.246 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:32:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:20 smithi045 ceph-mon[95976]: pgmap v1461: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 21679/127203 objects degraded (17.043%); 35 MiB/s, 560 objects/s recovering 2024-09-21T10:32:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:20 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:32:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:20 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/715770725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:32:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:20 smithi045 ceph-mon[95976]: osdmap e469: 8 total, 7 up, 7 in 2024-09-21T10:32:20.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:20.336 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:20.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:20.355 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:20 smithi032 ceph-mon[105748]: pgmap v1461: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 21679/127203 objects degraded (17.043%); 35 MiB/s, 560 objects/s recovering 2024-09-21T10:32:20.356 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:20 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:32:20.356 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:20 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/715770725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:32:20.356 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:20 smithi032 ceph-mon[105748]: osdmap e469: 8 total, 7 up, 7 in 2024-09-21T10:32:20.356 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:20 smithi032 ceph-mon[101299]: pgmap v1461: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 21679/127203 objects degraded (17.043%); 35 MiB/s, 560 objects/s recovering 2024-09-21T10:32:20.356 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:20 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:32:20.356 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:20 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/715770725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:32:20.356 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:20 smithi032 ceph-mon[101299]: osdmap e469: 8 total, 7 up, 7 in 2024-09-21T10:32:20.437 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:20.476 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:20.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:20.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:20.620 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:20.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:20.722 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:20.763 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:20.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:20.865 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:20.906 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:20.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:21.006 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:21.039 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:21.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:21.141 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:21.174 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:21.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:21.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:21.227 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:32:21.227 INFO:teuthology.orchestra.run.smithi032.stdout: "success": "" 2024-09-21T10:32:21.227 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:32:21.276 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:21.316 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:21.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:21.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:21.418 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:21.451 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:21.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:21.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:21.552 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:21.591 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:21.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:21.693 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:21.733 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:21.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:21.834 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:21.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:21.867 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:21.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:21.968 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:22.005 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:22.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:22.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:22.083 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:22 smithi032 ceph-mon[101299]: pgmap v1463: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 16371/117258 objects degraded (13.962%); 60 MiB/s, 952 objects/s recovering 2024-09-21T10:32:22.083 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:22 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 16371/117258 objects degraded (13.962%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:22.083 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:22 smithi032 ceph-mon[105748]: pgmap v1463: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 16371/117258 objects degraded (13.962%); 60 MiB/s, 952 objects/s recovering 2024-09-21T10:32:22.084 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:22 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 16371/117258 objects degraded (13.962%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:22.107 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:22.135 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:22.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:22.237 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:22.269 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:22.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:22 smithi045 ceph-mon[95976]: pgmap v1463: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 16371/117258 objects degraded (13.962%); 60 MiB/s, 952 objects/s recovering 2024-09-21T10:32:22.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:22 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 16371/117258 objects degraded (13.962%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:22.372 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:22.418 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:22.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:22.521 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:22.563 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:22.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:22.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:22.664 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:22.694 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:22.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:22.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:22.795 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:22.828 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:22.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:22.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:22.930 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:22.969 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:22.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:22.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:22.990 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:32:23.071 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:23.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:23.115 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:23.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:23.216 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:23.247 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:23.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:23.349 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:23.379 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:23.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:23.481 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:23.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:23.525 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:23.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:23.626 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:23.671 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:23.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:23.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:23.773 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:23.805 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:23.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:23.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:23.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:23.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:23.941 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:23.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:24.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:24.089 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:24.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:24.189 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:24.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:24 smithi032 ceph-mon[105748]: pgmap v1464: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 15882/114951 objects degraded (13.816%); 40 MiB/s, 648 objects/s recovering 2024-09-21T10:32:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:24 smithi032 ceph-mon[101299]: pgmap v1464: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 15882/114951 objects degraded (13.816%); 40 MiB/s, 648 objects/s recovering 2024-09-21T10:32:24.223 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:24.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:24.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:24.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:24 smithi045 ceph-mon[95976]: pgmap v1464: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 15882/114951 objects degraded (13.816%); 40 MiB/s, 648 objects/s recovering 2024-09-21T10:32:24.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:24.354 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:24.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:24.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:24.456 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:24.491 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:24.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:24.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:24.593 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:24.625 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:24.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:24.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:24.727 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:24.769 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:24.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:24.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:24.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:24.900 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:24.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:24.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:25.002 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:25.047 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:25.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:25.148 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:25.187 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:25.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:25.289 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:25.318 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:25.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:25.420 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:25.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:25.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:25.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:25.548 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:25.575 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:25.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:25.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:25.677 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:25.704 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:25.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:25.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:25.806 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:25.833 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:25.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:25.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:25.935 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:25.961 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:25.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:25.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:26.063 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:26.090 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:26.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:26.179 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:32:26.191 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:26.262 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:26.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:26.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:26.364 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:26.394 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:26.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:26 smithi032 ceph-mon[105748]: pgmap v1465: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 916 op/s; 13291/105774 objects degraded (12.565%); 42 MiB/s, 676 objects/s recovering 2024-09-21T10:32:26.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:26 smithi032 ceph-mon[105748]: osdmap e470: 8 total, 7 up, 7 in 2024-09-21T10:32:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:26 smithi032 ceph-mon[101299]: pgmap v1465: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 916 op/s; 13291/105774 objects degraded (12.565%); 42 MiB/s, 676 objects/s recovering 2024-09-21T10:32:26.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:26 smithi032 ceph-mon[101299]: osdmap e470: 8 total, 7 up, 7 in 2024-09-21T10:32:26.495 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:26.522 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:26.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:26.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:26.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:26 smithi045 ceph-mon[95976]: pgmap v1465: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 916 op/s; 13291/105774 objects degraded (12.565%); 42 MiB/s, 676 objects/s recovering 2024-09-21T10:32:26.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:26 smithi045 ceph-mon[95976]: osdmap e470: 8 total, 7 up, 7 in 2024-09-21T10:32:26.623 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:26.661 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:26.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:26.663 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:26.717 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:26.763 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:26.790 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:26.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:26.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:26.919 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:26.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:26.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:27.021 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:27.052 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:27.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:27.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:27.154 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:27.184 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:27.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:27.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:27 smithi032 ceph-mon[105748]: osdmap e471: 8 total, 7 up, 7 in 2024-09-21T10:32:27.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:27 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 13291/105774 objects degraded (12.565%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:27.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:27 smithi032 ceph-mon[105748]: osdmap e472: 8 total, 7 up, 7 in 2024-09-21T10:32:27.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:27 smithi032 ceph-mon[101299]: osdmap e471: 8 total, 7 up, 7 in 2024-09-21T10:32:27.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:27 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 13291/105774 objects degraded (12.565%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:27.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:27 smithi032 ceph-mon[101299]: osdmap e472: 8 total, 7 up, 7 in 2024-09-21T10:32:27.286 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:27.314 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:27.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:27.416 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:27.442 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:27.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:27.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:27.519 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4, 1] out_osds: [0] dead_osds: [0] live_osds: [1, 2, 3, 4, 6, 7, 5] 2024-09-21T10:32:27.519 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:32:27.519 INFO:tasks.thrashosds.thrasher:Reweighting osd 7 to 0.727892027614544 2024-09-21T10:32:27.520 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 7 0.727892027614544 2024-09-21T10:32:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:27 smithi045 ceph-mon[95976]: osdmap e471: 8 total, 7 up, 7 in 2024-09-21T10:32:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:27 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 13291/105774 objects degraded (12.565%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:27 smithi045 ceph-mon[95976]: osdmap e472: 8 total, 7 up, 7 in 2024-09-21T10:32:27.544 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:27.592 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:27.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:27.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:27.694 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:27.721 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:27.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:27.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:27.823 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:27.851 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:27.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:27.953 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:27.982 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:27.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:27.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:28.084 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:28.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:28.125 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:28.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:28.226 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:28.256 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:28.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:28.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:28.358 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:28.385 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:28.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:28.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:28 smithi032 ceph-mon[101299]: pgmap v1468: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 11316/100530 objects degraded (11.256%); 53 MiB/s, 854 objects/s recovering 2024-09-21T10:32:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:28 smithi032 ceph-mon[101299]: osdmap e473: 8 total, 7 up, 7 in 2024-09-21T10:32:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:28 smithi032 ceph-mon[105748]: pgmap v1468: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 11316/100530 objects degraded (11.256%); 53 MiB/s, 854 objects/s recovering 2024-09-21T10:32:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:28 smithi032 ceph-mon[105748]: osdmap e473: 8 total, 7 up, 7 in 2024-09-21T10:32:28.486 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:28.528 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:28.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:28.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:28 smithi045 ceph-mon[95976]: pgmap v1468: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 11316/100530 objects degraded (11.256%); 53 MiB/s, 854 objects/s recovering 2024-09-21T10:32:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:28 smithi045 ceph-mon[95976]: osdmap e473: 8 total, 7 up, 7 in 2024-09-21T10:32:28.630 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:28.670 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:28.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:28.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:28.772 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:28.816 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:28.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:28.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:28.917 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:28.954 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:28.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:28.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:29.055 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:29.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:29.101 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:29.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:29.202 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:29.235 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:29.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:29.337 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:29.376 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:29.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:29.421 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:29 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2487794098' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:29.422 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:29 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2487794098' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:29.422 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:32:29.422 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:32:29.422 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:32:29.422 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:32:29.423 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:32:29.478 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:29.518 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:29.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:29 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2487794098' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:29.620 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:29.643 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:29.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:29.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:32:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:32:29] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.51.0" 2024-09-21T10:32:29.744 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:29.784 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:29.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:29.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:29.886 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:29.911 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:29.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:29.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:30.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:30.055 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:30.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:30.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:30.157 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:30.188 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:30.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:30.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:30.290 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:30.320 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:30.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:30.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:30.422 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:30.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:30 smithi032 ceph-mon[101299]: pgmap v1471: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 436 op/s; 11316/100530 objects degraded (11.256%); 21 MiB/s, 330 objects/s recovering 2024-09-21T10:32:30.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:30 smithi032 ceph-mon[105748]: pgmap v1471: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 436 op/s; 11316/100530 objects degraded (11.256%); 21 MiB/s, 330 objects/s recovering 2024-09-21T10:32:30.467 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:30.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:30.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:30.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:30 smithi045 ceph-mon[95976]: pgmap v1471: 109 pgs: 4 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 436 op/s; 11316/100530 objects degraded (11.256%); 21 MiB/s, 330 objects/s recovering 2024-09-21T10:32:30.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:30.599 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:30.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:30.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:30.701 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:30.748 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:30.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:30.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:30.849 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:30.882 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:30.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:30.983 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:31.023 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:31.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:31.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:31.125 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:31.160 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:31.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:31.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:31.262 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:31.287 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:31.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:31.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:31.389 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:31.414 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:31.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:31.515 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:31.543 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:31.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:31.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:31.645 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:31.670 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:31.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:31.772 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:31.798 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:31.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:31.900 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:31.924 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:31.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:31.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:32.025 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:32.109 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:32.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:32.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:32.211 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:32.246 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:32.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:32.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:32.348 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:32.373 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:32.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:32 smithi032 ceph-mon[105748]: pgmap v1472: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 103 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 6829/94251 objects degraded (7.246%); 51 MiB/s, 810 objects/s recovering 2024-09-21T10:32:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:32 smithi032 ceph-mon[105748]: osdmap e474: 8 total, 7 up, 7 in 2024-09-21T10:32:32.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:32 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 6829/94251 objects degraded (7.246%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:32.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:32 smithi032 ceph-mon[101299]: pgmap v1472: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 103 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 6829/94251 objects degraded (7.246%); 51 MiB/s, 810 objects/s recovering 2024-09-21T10:32:32.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:32 smithi032 ceph-mon[101299]: osdmap e474: 8 total, 7 up, 7 in 2024-09-21T10:32:32.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:32 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 6829/94251 objects degraded (7.246%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:32.475 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:32.501 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:32.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:32.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:32 smithi045 ceph-mon[95976]: pgmap v1472: 109 pgs: 3 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 103 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 6829/94251 objects degraded (7.246%); 51 MiB/s, 810 objects/s recovering 2024-09-21T10:32:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:32 smithi045 ceph-mon[95976]: osdmap e474: 8 total, 7 up, 7 in 2024-09-21T10:32:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:32 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 6829/94251 objects degraded (7.246%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:32.604 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:32.629 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:32.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:32.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:32.731 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:32.755 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:32.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:32.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:32.857 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:32.882 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:32.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:32.984 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:33.009 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:33.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:33.110 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:33.139 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:33.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:33.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:33.241 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:33.278 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:33.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:33.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:33.380 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:33.426 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:33.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:33.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:33.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:33 smithi032 ceph-mon[101299]: osdmap e475: 8 total, 7 up, 7 in 2024-09-21T10:32:33.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:33 smithi032 ceph-mon[101299]: osdmap e476: 8 total, 7 up, 7 in 2024-09-21T10:32:33.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:33 smithi032 ceph-mon[105748]: osdmap e475: 8 total, 7 up, 7 in 2024-09-21T10:32:33.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:33 smithi032 ceph-mon[105748]: osdmap e476: 8 total, 7 up, 7 in 2024-09-21T10:32:33.528 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:33 smithi045 ceph-mon[95976]: osdmap e475: 8 total, 7 up, 7 in 2024-09-21T10:32:33.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:33 smithi045 ceph-mon[95976]: osdmap e476: 8 total, 7 up, 7 in 2024-09-21T10:32:33.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:33.571 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:33.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:33.672 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:33.707 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:33.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:33.808 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:33.846 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:33.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:33.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:33.948 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:33.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:33.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:33.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:34.091 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:34.127 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:34.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:34.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:34.229 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:34.274 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:34.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:34.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:34.376 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:34.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:34.415 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:34.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:34.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:34 smithi032 ceph-mon[105748]: pgmap v1475: 109 pgs: 1 peering, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 103 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 5426/92502 objects degraded (5.866%); 57 MiB/s, 917 objects/s recovering 2024-09-21T10:32:34.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:34.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:34 smithi032 ceph-mon[101299]: pgmap v1475: 109 pgs: 1 peering, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 103 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 5426/92502 objects degraded (5.866%); 57 MiB/s, 917 objects/s recovering 2024-09-21T10:32:34.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:34.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:34 smithi045 ceph-mon[95976]: pgmap v1475: 109 pgs: 1 peering, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 103 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 5426/92502 objects degraded (5.866%); 57 MiB/s, 917 objects/s recovering 2024-09-21T10:32:34.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:34.547 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:34.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:34.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:34.649 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:34.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:34.696 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:34.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:34.797 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:34.840 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:34.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:34.941 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:34.977 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:34.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:34.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:35.079 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:35.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:35.124 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:35.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:35.225 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:35.263 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:35.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:35.365 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:35.406 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:35.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:35.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:35.508 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:35.540 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:35.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:35.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:35.641 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:35.676 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:35.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:35.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:35.778 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:35.811 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:35.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:35.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:35.901 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:35.913 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:35.956 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:35.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:35.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:36.058 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:36.096 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:36.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:36.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:36.131 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:32:36.198 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:36.236 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:36.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:36.338 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:36.360 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:36.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:36.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:36 smithi032 ceph-mon[101299]: pgmap v1477: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 3 active+undersized+degraded+remapped+backfilling, 104 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 763 op/s; 1569/86790 objects degraded (1.808%); 92 MiB/s, 1.47k objects/s recovering 2024-09-21T10:32:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:36 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2694226997' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:32:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:36 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:32:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:36 smithi032 ceph-mon[105748]: pgmap v1477: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 3 active+undersized+degraded+remapped+backfilling, 104 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 763 op/s; 1569/86790 objects degraded (1.808%); 92 MiB/s, 1.47k objects/s recovering 2024-09-21T10:32:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:36 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2694226997' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:32:36.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:36 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:32:36.461 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:36.484 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:36.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:36.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:36 smithi045 ceph-mon[95976]: pgmap v1477: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 3 active+undersized+degraded+remapped+backfilling, 104 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 763 op/s; 1569/86790 objects degraded (1.808%); 92 MiB/s, 1.47k objects/s recovering 2024-09-21T10:32:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:36 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2694226997' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:32:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:36 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:32:36.585 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:36.609 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:36.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:36.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:36.711 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:36.734 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:36.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:36.836 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:36.860 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:36.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:36.962 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:36.989 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:36.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:36.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:37.091 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:37.128 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:37.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:37.230 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:37.256 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:37.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:37.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:37.358 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:37.381 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:37.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:37.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:37.404 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:32:37.404 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:32:37.404 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[101299]: osdmap e477: 8 total, 7 up, 7 in 2024-09-21T10:32:37.404 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[101299]: Health check update: Degraded data redundancy: 1569/86790 objects degraded (1.808%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:37.404 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[101299]: osdmap e478: 8 total, 7 up, 7 in 2024-09-21T10:32:37.404 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:32:37.404 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:32:37.404 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[105748]: osdmap e477: 8 total, 7 up, 7 in 2024-09-21T10:32:37.405 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[105748]: Health check update: Degraded data redundancy: 1569/86790 objects degraded (1.808%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:37.405 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:37 smithi032 ceph-mon[105748]: osdmap e478: 8 total, 7 up, 7 in 2024-09-21T10:32:37.424 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:32:37.424 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:32:37.483 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:37.505 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:37.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:37.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:37 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:32:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:37 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:32:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:37 smithi045 ceph-mon[95976]: osdmap e477: 8 total, 7 up, 7 in 2024-09-21T10:32:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:37 smithi045 ceph-mon[95976]: Health check update: Degraded data redundancy: 1569/86790 objects degraded (1.808%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:32:37.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:37 smithi045 ceph-mon[95976]: osdmap e478: 8 total, 7 up, 7 in 2024-09-21T10:32:37.606 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:37.628 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:37.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:37.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:37.730 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:37.752 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:37.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:37.853 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:37.876 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:37.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:37.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:37.977 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:38.000 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:38.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:38.102 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:38.132 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:38.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:38.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:38.234 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:38.266 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:38.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:38.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:38.367 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:38.389 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:38.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:38.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:38.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:38 smithi032 ceph-mon[101299]: pgmap v1479: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 106 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 1458/83097 objects degraded (1.755%); 55 MiB/s, 876 objects/s recovering 2024-09-21T10:32:38.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:38 smithi032 ceph-mon[101299]: osdmap e479: 8 total, 7 up, 7 in 2024-09-21T10:32:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:38 smithi032 ceph-mon[105748]: pgmap v1479: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 106 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 1458/83097 objects degraded (1.755%); 55 MiB/s, 876 objects/s recovering 2024-09-21T10:32:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:38 smithi032 ceph-mon[105748]: osdmap e479: 8 total, 7 up, 7 in 2024-09-21T10:32:38.490 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:38.512 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:38.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:38.534 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:38 smithi045 ceph-mon[95976]: pgmap v1479: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 106 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 1458/83097 objects degraded (1.755%); 55 MiB/s, 876 objects/s recovering 2024-09-21T10:32:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:38 smithi045 ceph-mon[95976]: osdmap e479: 8 total, 7 up, 7 in 2024-09-21T10:32:38.614 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:38.637 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:38.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:38.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:38.738 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:38.762 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:38.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:38.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:38.864 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:38.886 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:38.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:38.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:38.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:39.012 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:39.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:39.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:39.114 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:39.148 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:39.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:39.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:39.250 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:39.284 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:39.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:39.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:39.314 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:32:39.314 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:32:39.314 INFO:teuthology.orchestra.run.smithi032.stdout: "complaint_time": 30, 2024-09-21T10:32:39.314 INFO:teuthology.orchestra.run.smithi032.stdout: "num_blocked_ops": 0 2024-09-21T10:32:39.315 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:32:39.386 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:39.421 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:39.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:39.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:39.425 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:32:39.523 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:39.566 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:39.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:39.668 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:39.712 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:39.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:39.814 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:39.857 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:39.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:39.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:39.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:32:39.880 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:32:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:32:39] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.51.0" 2024-09-21T10:32:39.960 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:40.004 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:40.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:40.084 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:40.105 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:40.137 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:40.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:40.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:40.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:39 smithi032 ceph-mon[101299]: pgmap v1482: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 106 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 1458/83097 objects degraded (1.755%); 40 MiB/s, 634 objects/s recovering 2024-09-21T10:32:40.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:39 smithi032 ceph-mon[101299]: osdmap e480: 8 total, 7 up, 7 in 2024-09-21T10:32:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:39 smithi032 ceph-mon[105748]: pgmap v1482: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 106 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 1458/83097 objects degraded (1.755%); 40 MiB/s, 634 objects/s recovering 2024-09-21T10:32:40.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:39 smithi032 ceph-mon[105748]: osdmap e480: 8 total, 7 up, 7 in 2024-09-21T10:32:40.238 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:40.273 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:40.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:40.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:39 smithi045 ceph-mon[95976]: pgmap v1482: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 106 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 1458/83097 objects degraded (1.755%); 40 MiB/s, 634 objects/s recovering 2024-09-21T10:32:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:39 smithi045 ceph-mon[95976]: osdmap e480: 8 total, 7 up, 7 in 2024-09-21T10:32:40.375 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:40.414 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:40.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:40.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:40.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:40.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:40.561 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:40.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:40.583 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:32:40.661 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:40.700 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:40.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:40.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:40.805 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:40.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:40.842 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:40.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:40.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:40.976 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:40.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:40.993 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.7 to 0.727892 (ba57) 2024-09-21T10:32:41.078 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:41.107 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:41.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:41.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:41.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:41.234 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:41.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:41.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:40 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3489096625' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.727892027614544} : dispatch 2024-09-21T10:32:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:40 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.727892027614544} : dispatch 2024-09-21T10:32:41.336 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:41.352 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:40 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3489096625' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.727892027614544} : dispatch 2024-09-21T10:32:41.352 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:40 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.727892027614544} : dispatch 2024-09-21T10:32:41.352 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:40 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3489096625' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.727892027614544} : dispatch 2024-09-21T10:32:41.353 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:40 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 0.727892027614544} : dispatch 2024-09-21T10:32:41.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:41.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:41.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:41.460 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:41.482 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:41.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:41.583 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:41.605 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:41.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:41.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:41.729 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:41.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:41.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:41.831 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:41.856 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:41.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:41.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:41.988 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:41.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:41.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:42.090 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:42.113 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:42.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:42.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:42.214 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:42.241 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:42.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:42.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:41 smithi045 ceph-mon[95976]: pgmap v1484: 109 pgs: 109 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 19 MiB/s, 298 objects/s recovering 2024-09-21T10:32:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:41 smithi045 ceph-mon[95976]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1458/83097 objects degraded (1.755%), 3 pgs degraded) 2024-09-21T10:32:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:41 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.727892027614544}]': finished 2024-09-21T10:32:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:41 smithi045 ceph-mon[95976]: osdmap e481: 8 total, 7 up, 7 in 2024-09-21T10:32:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:41 smithi045 ceph-mon[95976]: osdmap e482: 8 total, 7 up, 7 in 2024-09-21T10:32:42.287 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[101299]: pgmap v1484: 109 pgs: 109 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 19 MiB/s, 298 objects/s recovering 2024-09-21T10:32:42.287 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[101299]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1458/83097 objects degraded (1.755%), 3 pgs degraded) 2024-09-21T10:32:42.287 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.727892027614544}]': finished 2024-09-21T10:32:42.287 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[101299]: osdmap e481: 8 total, 7 up, 7 in 2024-09-21T10:32:42.287 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[101299]: osdmap e482: 8 total, 7 up, 7 in 2024-09-21T10:32:42.287 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[105748]: pgmap v1484: 109 pgs: 109 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 19 MiB/s, 298 objects/s recovering 2024-09-21T10:32:42.287 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[105748]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1458/83097 objects degraded (1.755%), 3 pgs degraded) 2024-09-21T10:32:42.287 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.727892027614544}]': finished 2024-09-21T10:32:42.288 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[105748]: osdmap e481: 8 total, 7 up, 7 in 2024-09-21T10:32:42.288 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:41 smithi032 ceph-mon[105748]: osdmap e482: 8 total, 7 up, 7 in 2024-09-21T10:32:42.342 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:42.369 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:42.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:42.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:42.470 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:42.496 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:42.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:42.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:42.598 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:42.623 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:42.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:42.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:42.751 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:42.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:42.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:42.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:42.879 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:42.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:42.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:42.981 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:43.007 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:43.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:43.109 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:43.132 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:43.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:43.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:43.233 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:43.262 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:43.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:43.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:43.364 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:43.407 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:43.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:43.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:43.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:43.551 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:43.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:43.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:43.654 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:43.699 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:43.699 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:43.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:43.767 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:43.801 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:43.841 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:43.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:43.942 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:43.975 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:43.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:43.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:44.077 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:44.118 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:44.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:44.220 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:44.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:44.268 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:44.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:43 smithi045 ceph-mon[95976]: pgmap v1487: 109 pgs: 2 peering, 107 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 18 MiB/s, 288 objects/s recovering 2024-09-21T10:32:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:43 smithi045 ceph-mon[95976]: osdmap e483: 8 total, 7 up, 7 in 2024-09-21T10:32:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:43 smithi045 ceph-mon[95976]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 480'13019 2024-09-21T10:32:44.290 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:43 smithi032 ceph-mon[105748]: pgmap v1487: 109 pgs: 2 peering, 107 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 18 MiB/s, 288 objects/s recovering 2024-09-21T10:32:44.290 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:43 smithi032 ceph-mon[105748]: osdmap e483: 8 total, 7 up, 7 in 2024-09-21T10:32:44.290 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:43 smithi032 ceph-mon[105748]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 480'13019 2024-09-21T10:32:44.290 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:43 smithi032 ceph-mon[101299]: pgmap v1487: 109 pgs: 2 peering, 107 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 18 MiB/s, 288 objects/s recovering 2024-09-21T10:32:44.290 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:43 smithi032 ceph-mon[101299]: osdmap e483: 8 total, 7 up, 7 in 2024-09-21T10:32:44.290 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:43 smithi032 ceph-mon[101299]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 480'13019 2024-09-21T10:32:44.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:44.398 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:44.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:44.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:44.500 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:44.530 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:44.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:44.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:44.673 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:44.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:44.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:44.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:44.813 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:44.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:44.915 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:44.948 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:44.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:44.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:45.050 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:45.082 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:45.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:45.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:45.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:45.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:44 smithi032 ceph-mon[101299]: osdmap e484: 8 total, 7 up, 7 in 2024-09-21T10:32:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:44 smithi032 ceph-mon[101299]: osdmap e485: 8 total, 7 up, 7 in 2024-09-21T10:32:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:44 smithi032 ceph-mon[105748]: osdmap e484: 8 total, 7 up, 7 in 2024-09-21T10:32:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:44 smithi032 ceph-mon[105748]: osdmap e485: 8 total, 7 up, 7 in 2024-09-21T10:32:45.216 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:45.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:45.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:44 smithi045 ceph-mon[95976]: osdmap e484: 8 total, 7 up, 7 in 2024-09-21T10:32:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:44 smithi045 ceph-mon[95976]: osdmap e485: 8 total, 7 up, 7 in 2024-09-21T10:32:45.317 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:45.348 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:45.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:45.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:45.450 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:45.485 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:45.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:45.587 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:45.624 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:45.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:45.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:45.725 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:45.755 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:45.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:45.856 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:45.889 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:45.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:45.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:45.991 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:46.026 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:46.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:46.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:46.128 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:46.159 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:46.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:46.260 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:45 smithi045 ceph-mon[95976]: pgmap v1490: 109 pgs: 5 peering, 104 active+clean; 1.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 0 B/s, 1 objects/s recovering 2024-09-21T10:32:46.301 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:46.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:46.353 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:45 smithi032 ceph-mon[105748]: pgmap v1490: 109 pgs: 5 peering, 104 active+clean; 1.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 0 B/s, 1 objects/s recovering 2024-09-21T10:32:46.353 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:45 smithi032 ceph-mon[101299]: pgmap v1490: 109 pgs: 5 peering, 104 active+clean; 1.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 0 B/s, 1 objects/s recovering 2024-09-21T10:32:46.402 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:46.447 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:46.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:46.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:46.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:46.586 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:46.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:46.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:46.687 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:46.724 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:46.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout: "size": 20, 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 600, 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [ 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45848.0:142517 9.1 9:8cf625f9:::benchmark_data_smithi032_265250_object14715:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:26:22.106076+0000", 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 384.70872102099997, 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.28117989300000001, 2024-09-21T10:32:46.819 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45848", 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 142517 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.106076+0000", 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.820 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.106076+0000", 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.106078+0000", 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.8679999999999999e-06 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.106083+0000", 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6469999999999997e-06 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.821 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.106093+0000", 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.0127e-05 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.106100+0000", 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4969999999999998e-06 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.106169+0000", 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.957e-05 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.106296+0000", 2024-09-21T10:32:46.822 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012710999999999999 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.106420+0000", 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000123707 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.110414+0000", 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003994109 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.110426+0000", 2024-09-21T10:32:46.823 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1897e-05 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.387141+0000", 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27671528099999998 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.387209+0000", 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7479999999999998e-05 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.824 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.387256+0000", 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7599999999999998e-05 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45848.0:142535 9.1 9:8b698986:::benchmark_data_smithi032_265250_object14733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:26:22.132972+0000", 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 384.68182507199998, 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.25443928999999998, 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45848", 2024-09-21T10:32:46.825 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 142535 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.132972+0000", 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.132972+0000", 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.826 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.132973+0000", 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1200000000000001e-06 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.132977+0000", 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9349999999999996e-06 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.132990+0000", 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2381000000000001e-05 2024-09-21T10:32:46.827 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.132996+0000", 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.3049999999999998e-06 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.138764+0000", 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0057681069999999997 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.138903+0000", 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013847100000000001 2024-09-21T10:32:46.828 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.139052+0000", 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014941499999999999 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.144673+0000", 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0056209040000000004 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.144703+0000", 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.9964e-05 2024-09-21T10:32:46.829 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.387305+0000", 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.242601766 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.387365+0000", 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0523000000000003e-05 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:22.387412+0000", 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6399000000000001e-05 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.830 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45848.0:156668 9.8 9:12be7a27:::benchmark_data_smithi032_265250_object28866:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:26:40.078468+0000", 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 366.736329582, 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.18413883, 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45848", 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:32:46.831 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 156668 2024-09-21T10:32:46.832 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.832 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.078468+0000", 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.078468+0000", 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.833 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.078468+0000", 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.9099999999999999e-07 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.078470+0000", 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.939e-06 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.078476+0000", 2024-09-21T10:32:46.834 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0410000000000003e-06 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.078481+0000", 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.0549999999999999e-06 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.099655+0000", 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.021173328000000002 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.099815+0000", 2024-09-21T10:32:46.835 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00016053500000000001 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.099972+0000", 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015699099999999999 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.108436+0000", 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0084640119999999999 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.113309+0000", 2024-09-21T10:32:46.836 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0048730730000000003 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.258769+0000", 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.14546023299999999 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.258904+0000", 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013445500000000001 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:26:40.262607+0000", 2024-09-21T10:32:46.837 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.003702777 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45848.0:184523 9.2 9:43842619:::benchmark_data_smithi032_265250_object56721:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:15.116507+0000", 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 331.69829037800002, 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.30559770800000002, 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:46.838 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45848", 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 184523 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116507+0000", 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116507+0000", 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.839 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116508+0000", 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.423e-06 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116512+0000", 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8569999999999997e-06 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116524+0000", 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.128e-05 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.840 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116531+0000", 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7980000000000006e-06 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116654+0000", 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012254900000000001 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116783+0000", 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012886199999999999 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:32:46.841 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116905+0000", 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00012229799999999999 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.124785+0000", 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0078801909999999999 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.127511+0000", 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0027255539999999998 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.421963+0000", 2024-09-21T10:32:46.842 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.29445197499999998 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422057+0000", 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.4258000000000005e-05 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422105+0000", 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7663000000000001e-05 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.843 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45848.0:184534 9.2 9:41fb7d7c:::benchmark_data_smithi032_265250_object56732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:15.116926+0000", 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 331.69787159100002, 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.30533924499999998, 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45848", 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 184534 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.844 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116926+0000", 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116926+0000", 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116927+0000", 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.8899999999999998e-07 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.845 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116931+0000", 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.946e-06 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116939+0000", 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.8130000000000002e-06 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.116947+0000", 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1380000000000001e-06 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.846 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.120541+0000", 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0035942510000000001 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.120696+0000", 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000154792 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.120829+0000", 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00013382300000000001 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:46.847 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.124863+0000", 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0040337189999999998 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.127616+0000", 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0027524030000000001 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422173+0000", 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.29455787100000003 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.848 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422220+0000", 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6798999999999997e-05 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422265+0000", 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4901000000000003e-05 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_op(client.45848.0:184543 9.2 9:497fc370:::benchmark_data_smithi032_265250_object56741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:15.138056+0000", 2024-09-21T10:32:46.849 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 331.67674155700001, 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.28437590600000001, 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "client_info": { 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "client": "client.45848", 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "tid": 184543 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.138056+0000", 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.850 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.138056+0000", 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.138058+0000", 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.759e-06 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.138064+0000", 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5950000000000004e-06 2024-09-21T10:32:46.851 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.138079+0000", 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4554e-05 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.138087+0000", 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9829999999999996e-06 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.138154+0000", 2024-09-21T10:32:46.852 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7057000000000005e-05 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.138296+0000", 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014246899999999999 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.138446+0000", 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015010499999999999 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "op_commit", 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.143591+0000", 2024-09-21T10:32:46.853 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.005144376 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422142+0000", 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27855138499999998 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422316+0000", 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00017388000000000001 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.854 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422395+0000", 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9345999999999996e-05 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422432+0000", 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.6396999999999997e-05 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.45848.0:184544 9.4 e403/378 9:232de4a7:::benchmark_data_smithi032_265250_object56742:head v 403'11536, mlcod=403'11533)", 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:15.146362+0000", 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 331.668435069, 2024-09-21T10:32:46.855 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27961903999999999, 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146362+0000", 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146362+0000", 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.856 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146363+0000", 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.128e-06 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146372+0000", 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.4419999999999998e-06 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146387+0000", 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4654999999999999e-05 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.857 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146394+0000", 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7449999999999995e-06 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146462+0000", 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.7397000000000005e-05 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.858 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146485+0000", 2024-09-21T10:32:46.859 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.3161999999999999e-05 2024-09-21T10:32:46.859 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.859 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.859 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.859 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.425951+0000", 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27946631199999999 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.425981+0000", 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0199e-05 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.45848.0:184523 9.2 e403/378 ondisk, result = 0)", 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:15.146801+0000", 2024-09-21T10:32:46.860 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 331.66799614000001, 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.275308738, 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146801+0000", 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146801+0000", 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.861 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146802+0000", 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.1399999999999996e-07 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146806+0000", 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4270000000000001e-06 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.862 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.146816+0000", 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.006e-05 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.421480+0000", 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27466328699999998 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.421949+0000", 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00046888700000000003 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.863 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.421957+0000", 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9459999999999998e-06 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422110+0000", 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00015341700000000001 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop_reply(client.45848.0:184543 9.2 e403/378 ondisk, result = 0)", 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:27:15.147677+0000", 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 331.66712003200001, 2024-09-21T10:32:46.864 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27447096199999999, 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.147677+0000", 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.147677+0000", 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.865 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.147678+0000", 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.0800000000000004e-07 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.147684+0000", 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.806e-06 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.421471+0000", 2024-09-21T10:32:46.866 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.27378714399999998 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.421488+0000", 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6537000000000001e-05 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422129+0000", 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000641711 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.867 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422137+0000", 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.3640000000000001e-06 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:27:15.422148+0000", 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1592e-05 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:189603 10.d e466/463 10:b05e94c9:::benchmark_data_smithi032_297285_object72141:head v 466'11902, mlcod=466'11899)", 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:11.797437+0000", 2024-09-21T10:32:46.868 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 35.017360771, 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.046397815000000002, 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.797437+0000", 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.797437+0000", 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.869 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.797437+0000", 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4899999999999995e-07 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.797443+0000", 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3489999999999999e-06 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.797456+0000", 2024-09-21T10:32:46.870 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3461e-05 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.797463+0000", 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8369999999999999e-06 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.807786+0000", 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.010323031999999999 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.807809+0000", 2024-09-21T10:32:46.871 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2697999999999999e-05 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.843594+0000", 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.035785063999999998 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:11.843834+0000", 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00024072500000000001 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.872 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:203909 10.4 e473/470 10:209ccf70:::benchmark_data_smithi032_297285_object86447:head v 473'12885, mlcod=473'12883)", 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:29.618821+0000", 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 17.195975951000001, 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.078076683999999993, 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.618821+0000", 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.618821+0000", 2024-09-21T10:32:46.873 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.618822+0000", 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.5600000000000005e-07 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.618827+0000", 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7990000000000001e-06 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.874 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.618835+0000", 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.1790000000000008e-06 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.618840+0000", 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.4710000000000001e-06 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.618885+0000", 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.5405999999999999e-05 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.618898+0000", 2024-09-21T10:32:46.875 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2498e-05 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.696819+0000", 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.077921564999999998 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.696898+0000", 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.9010000000000004e-05 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.876 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:203911 10.c e473/472 10:3435322f:::benchmark_data_smithi032_297285_object86449:head v 473'12608, mlcod=473'12602)", 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:29.620888+0000", 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 17.193909063, 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.076154982999999996, 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.620888+0000", 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.877 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.620888+0000", 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.620889+0000", 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.3799999999999997e-07 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.620894+0000", 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.8389999999999997e-06 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.878 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.620906+0000", 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2456999999999999e-05 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.620913+0000", 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4509999999999997e-06 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.620956+0000", 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.3050999999999998e-05 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.879 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.620972+0000", 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.6682999999999999e-05 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.696902+0000", 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.075929459000000005 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.697043+0000", 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00014150499999999999 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.880 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:203912 10.7 e473/419 10:ef56125f:::benchmark_data_smithi032_297285_object86450:head v 473'12641, mlcod=473'12639)", 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:29.621278+0000", 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 17.19351949, 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.088561928999999998, 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.621278+0000", 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.881 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.621278+0000", 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.621279+0000", 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8899999999999999e-07 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.621284+0000", 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.5620000000000003e-06 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.882 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.696463+0000", 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.075179241999999993 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.696476+0000", 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.2751999999999999e-05 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.696763+0000", 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00028728100000000002 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.696788+0000", 2024-09-21T10:32:46.883 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.4100999999999998e-05 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.709397+0000", 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.01260904 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:29.709840+0000", 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00044326199999999999 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.884 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "MOSDPGPush(10.d 473/463 [PushOp(10:ba0aef28:::benchmark_data_smithi032_297285_object109595:head, version: 448'6887, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:ba0aef28:::benchmark_data_smithi032_297285_object109595:head@448'6887, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:30.511369+0000", 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 16.30342868, 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.064697260000000006, 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "started", 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.511369+0000", 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.886 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.511369+0000", 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.511370+0000", 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.4799999999999997e-07 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.511416+0000", 2024-09-21T10:32:46.887 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6078999999999997e-05 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.574998+0000", 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.063582646000000007 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.575029+0000", 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.0611999999999998e-05 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.575826+0000", 2024-09-21T10:32:46.888 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00079752700000000002 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.575840+0000", 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.3567000000000001e-05 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.576066+0000", 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00022608099999999999 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.889 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:204675 10.4 e473/470 10:2708fb1f:::benchmark_data_smithi032_297285_object87213:head v 473'12933, mlcod=473'12931)", 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:30.511918+0000", 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 16.302879367999999, 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.066816098000000004, 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.511918+0000", 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.890 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.511918+0000", 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.511919+0000", 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.6700000000000003e-07 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.511928+0000", 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.7479999999999999e-06 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.891 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.573679+0000", 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.061750961 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.573701+0000", 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.2424000000000001e-05 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.573816+0000", 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000114938 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.892 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.573843+0000", 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6889e-05 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.578601+0000", 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0047581439999999997 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:30.578734+0000", 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.000133227 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.893 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:214638 10.5 e485/482 10:a91e3883:::benchmark_data_smithi032_297285_object97176:head v 485'13219, mlcod=485'13216)", 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:46.268515+0000", 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.54628277999999997, 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.053887984, 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268515+0000", 2024-09-21T10:32:46.894 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268515+0000", 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268516+0000", 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.043e-06 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.895 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268522+0000", 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.5989999999999998e-06 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268534+0000", 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1739999999999999e-05 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268542+0000", 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.695e-06 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.896 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.310996+0000", 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.042454641000000001 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.311023+0000", 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.6258000000000002e-05 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.322356+0000", 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.011333602999999999 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.897 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.322403+0000", 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.6405e-05 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:214640 10.2 e485/467 10:4cd58278:::benchmark_data_smithi032_297285_object97178:head v 485'13492, mlcod=485'13490)", 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:46.268563+0000", 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.54623485500000002, 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.045983569000000002, 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.898 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268563+0000", 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268563+0000", 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268563+0000", 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.1999999999999998e-07 2024-09-21T10:32:46.899 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268570+0000", 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.46e-06 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268582+0000", 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.1609e-05 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268588+0000", 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4350000000000002e-06 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.900 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268640+0000", 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 5.2309999999999999e-05 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.268661+0000", 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.0301000000000001e-05 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.314498+0000", 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.045837787999999997 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.901 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.314546+0000", 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.7746e-05 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:214636 10.c e485/472 10:399c87fb:::benchmark_data_smithi032_297285_object97174:head v 485'13307, mlcod=485'13305)", 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:46.269134+0000", 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.54566378800000004, 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.050549841999999998, 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.902 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.269134+0000", 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.269134+0000", 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.903 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.269134+0000", 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 7.7700000000000004e-07 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.269141+0000", 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.1349999999999997e-06 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.310704+0000", 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.041563711000000003 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.904 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.310719+0000", 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.4849e-05 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.311124+0000", 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.00040472400000000002 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.311143+0000", 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.9548e-05 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.905 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.319645+0000", 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0085020089999999996 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.319683+0000", 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 3.8089000000000001e-05 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:214642 10.5 e485/482 10:a6fd6090:::benchmark_data_smithi032_297285_object97180:head v 485'13220, mlcod=485'13216)", 2024-09-21T10:32:46.906 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:46.272492+0000", 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.54230569799999995, 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.049957793, 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.272492+0000", 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.272492+0000", 2024-09-21T10:32:46.907 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.272492+0000", 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.8800000000000002e-07 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.272499+0000", 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.4910000000000002e-06 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.310708+0000", 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.038209509000000003 2024-09-21T10:32:46.908 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.310720+0000", 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.148e-05 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.314744+0000", 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0040243689999999999 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.314765+0000", 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 2.1202000000000001e-05 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.909 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.322408+0000", 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0076423699999999999 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.322450+0000", 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 4.1684000000000003e-05 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: "description": "osd_repop(client.36234.0:214641 10.4 e485/470 10:2dcb3463:::benchmark_data_smithi032_297285_object97179:head v 485'13529, mlcod=485'13527)", 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: "initiated_at": "2024-09-21T10:32:46.272544+0000", 2024-09-21T10:32:46.910 INFO:teuthology.orchestra.run.smithi032.stdout: "age": 0.54225372699999996, 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.047235798000000002, 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "continuous": false, 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "type_data": { 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "events": [ 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "initiated", 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.272544+0000", 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "header_read", 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.272544+0000", 2024-09-21T10:32:46.911 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "throttled", 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.272545+0000", 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.78e-07 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "all_read", 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.272551+0000", 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 6.0580000000000003e-06 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "dispatched", 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.310797+0000", 2024-09-21T10:32:46.912 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.038246334 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "queued_for_pg", 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.310805+0000", 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 8.3489999999999992e-06 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "reached_pg", 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.314303+0000", 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0034975689999999999 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "started", 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.314320+0000", 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 1.7218e-05 2024-09-21T10:32:46.913 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "commit_sent", 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.319687+0000", 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 0.0053672190000000003 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: { 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: "event": "done", 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: "time": "2024-09-21T10:32:46.319780+0000", 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: "duration": 9.2072999999999998e-05 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-09-21T10:32:46.914 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-09-21T10:32:46.915 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:32:46.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:46.916 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:46.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:47.017 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:47.051 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:47.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:47.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:47.193 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:47.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:47.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:47 smithi045 ceph-mon[95976]: 1.0 scrub starts 2024-09-21T10:32:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:47 smithi045 ceph-mon[95976]: 1.0 scrub ok 2024-09-21T10:32:47.295 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:47.328 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:47.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:47.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:47.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:47.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:47 smithi032 ceph-mon[101299]: 1.0 scrub starts 2024-09-21T10:32:47.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:47 smithi032 ceph-mon[101299]: 1.0 scrub ok 2024-09-21T10:32:47.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:47 smithi032 ceph-mon[105748]: 1.0 scrub starts 2024-09-21T10:32:47.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:47 smithi032 ceph-mon[105748]: 1.0 scrub ok 2024-09-21T10:32:47.470 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:47.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:47.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:47.571 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:47.604 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:47.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:47.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:47.705 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:47.746 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:47.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:47.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:47.848 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:47.888 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:47.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:47.989 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:48.034 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:48.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:48.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:48.105 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4, 1] out_osds: [0] dead_osds: [0] live_osds: [1, 2, 3, 4, 6, 7, 5] 2024-09-21T10:32:48.106 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:32:48.106 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-21T10:32:48.106 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:32:48.106 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:32:48.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-21T10:32:48.136 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:48.180 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:48.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:48.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:48.237 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:32:48.277 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:48 smithi045 ceph-mon[95976]: pgmap v1492: 109 pgs: 1 active+remapped+backfilling, 4 peering, 104 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 1144/65583 objects misplaced (1.744%); 2.9 MiB/s, 48 objects/s recovering 2024-09-21T10:32:48.283 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:48.332 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:48 smithi032 ceph-mon[105748]: pgmap v1492: 109 pgs: 1 active+remapped+backfilling, 4 peering, 104 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 1144/65583 objects misplaced (1.744%); 2.9 MiB/s, 48 objects/s recovering 2024-09-21T10:32:48.333 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:48 smithi032 ceph-mon[101299]: pgmap v1492: 109 pgs: 1 active+remapped+backfilling, 4 peering, 104 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 1144/65583 objects misplaced (1.744%); 2.9 MiB/s, 48 objects/s recovering 2024-09-21T10:32:48.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:48.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:48.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:48.439 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:48.470 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:48.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:48.572 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:48.605 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:48.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:48.619 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:32:48.621 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:32:48.650 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:32:48.651 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:32:48.676 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:32:48.676 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:32:48.702 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:32:48.702 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:32:48.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:48.736 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:32:48.737 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:32:48.738 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:48.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:48.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:48.770 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:32:48.770 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:32:48.799 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:32:48.799 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:32:48.826 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:32:48.826 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:32:48.840 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:48.873 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:48.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:48.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:48.975 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:49.015 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:49.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:49.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:49.116 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:49.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:49.171 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:49.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:49.272 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:49.313 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:49.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:49.415 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:49.449 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:49.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:49.551 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:49.578 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:49.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:49.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:49.680 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:49.706 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:49.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:49.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:49.809 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:49.849 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:49.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:49.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:49.885 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:32:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:32:49] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.51.0" 2024-09-21T10:32:49.885 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:49.885 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:49.885 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:49 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1766777704' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:49.885 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:49 smithi032 ceph-mon[105748]: pgmap v1493: 109 pgs: 1 active+remapped+backfilling, 4 peering, 104 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 1144/65583 objects misplaced (1.744%); 2.3 MiB/s, 38 objects/s recovering 2024-09-21T10:32:49.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:49.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:49.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:49 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1766777704' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:49.886 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:49 smithi032 ceph-mon[101299]: pgmap v1493: 109 pgs: 1 active+remapped+backfilling, 4 peering, 104 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 1144/65583 objects misplaced (1.744%); 2.3 MiB/s, 38 objects/s recovering 2024-09-21T10:32:49.920 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:49.922 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:32:49.923 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:32:49.951 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:49.973 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:32:49.974 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:32:49.974 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:49 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1766777704' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:49.974 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:49 smithi045 ceph-mon[95976]: pgmap v1493: 109 pgs: 1 active+remapped+backfilling, 4 peering, 104 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 1144/65583 objects misplaced (1.744%); 2.3 MiB/s, 38 objects/s recovering 2024-09-21T10:32:49.987 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:49.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:49.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:50.089 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:50.132 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:50.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:50.233 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:50.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:50.267 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:50.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:50.368 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:50.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:50.401 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:50.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:50.502 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:50.545 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:50.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:50.647 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:50.678 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:50.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:50.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:50.780 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:50.801 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:50.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:50.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:50.903 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:50.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:50.941 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:50.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:50.956 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:50 smithi032 ceph-mon[105748]: 1.2 scrub starts 2024-09-21T10:32:50.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:50 smithi032 ceph-mon[101299]: 1.2 scrub starts 2024-09-21T10:32:50.978 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:50 smithi045 ceph-mon[95976]: 1.2 scrub starts 2024-09-21T10:32:51.042 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:51.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:51.093 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:51.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:51.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:51.221 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:51.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:51.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:51.323 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:51.354 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:51.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:51.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:51.457 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:51.495 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:51.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:51.562 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:32:51.596 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:51.618 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:51.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:51.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:51.719 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:51.758 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:51.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:51 smithi045 ceph-mon[95976]: pgmap v1494: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.1 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 981 op/s; 326/51867 objects misplaced (0.629%); 6.9 MiB/s, 111 objects/s recovering 2024-09-21T10:32:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:51 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1978595030' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:32:51.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:51 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:32:51.807 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:51 smithi032 ceph-mon[101299]: pgmap v1494: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.1 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 981 op/s; 326/51867 objects misplaced (0.629%); 6.9 MiB/s, 111 objects/s recovering 2024-09-21T10:32:51.807 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:51 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1978595030' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:32:51.807 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:51 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:32:51.807 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:51 smithi032 ceph-mon[105748]: pgmap v1494: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.1 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 981 op/s; 326/51867 objects misplaced (0.629%); 6.9 MiB/s, 111 objects/s recovering 2024-09-21T10:32:51.807 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:51 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1978595030' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:32:51.807 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:51 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:32:51.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:51.892 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:51.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:51.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:51.923 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:32:51.993 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:52.025 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:52.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:52.127 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:52.166 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:52.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:52.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:52.268 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:52.294 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:52.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:52.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:52.395 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:52.426 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:52.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:52.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:52.527 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:52.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:52.568 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:52.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:52.669 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:52.691 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:52.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:52.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:52.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:52 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:32:52.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:52 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:32:52.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:52 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:32:52.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:52 smithi045 ceph-mon[95976]: osdmap e486: 8 total, 7 up, 7 in 2024-09-21T10:32:52.792 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:52.820 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:52.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:52.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:52.854 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:52 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:32:52.854 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:52 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:32:52.854 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:52 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:32:52.854 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:52 smithi032 ceph-mon[105748]: osdmap e486: 8 total, 7 up, 7 in 2024-09-21T10:32:52.854 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:52 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:32:52.854 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:52 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:32:52.854 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:52 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:32:52.854 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:52 smithi032 ceph-mon[101299]: osdmap e486: 8 total, 7 up, 7 in 2024-09-21T10:32:52.922 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:52.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:52.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:52.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:53.058 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:53.080 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:53.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:53.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:53.118 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:32:53.118 INFO:teuthology.orchestra.run.smithi032.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:32:53.118 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:32:53.124 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:53.182 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:53.203 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:53.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:53.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:53.305 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:53.327 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:53.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:53.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:53.429 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:53.451 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:53.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:53.553 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:53.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:53.578 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:53.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:53.679 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:53.701 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:53.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:53.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:53.803 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:53.844 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:53.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:53.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:53.901 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:53 smithi045 ceph-mon[95976]: osdmap e487: 8 total, 7 up, 7 in 2024-09-21T10:32:53.901 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:53 smithi045 ceph-mon[95976]: pgmap v1497: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 326/49917 objects misplaced (0.653%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:32:53.901 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:53 smithi045 ceph-mon[95976]: 1.2 deep-scrub starts 2024-09-21T10:32:53.901 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:53 smithi045 ceph-mon[95976]: 1.2 deep-scrub ok 2024-09-21T10:32:53.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:53 smithi032 ceph-mon[101299]: osdmap e487: 8 total, 7 up, 7 in 2024-09-21T10:32:53.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:53 smithi032 ceph-mon[101299]: pgmap v1497: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 326/49917 objects misplaced (0.653%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:32:53.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:53 smithi032 ceph-mon[101299]: 1.2 deep-scrub starts 2024-09-21T10:32:53.931 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:53 smithi032 ceph-mon[101299]: 1.2 deep-scrub ok 2024-09-21T10:32:53.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:53 smithi032 ceph-mon[105748]: osdmap e487: 8 total, 7 up, 7 in 2024-09-21T10:32:53.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:53 smithi032 ceph-mon[105748]: pgmap v1497: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 326/49917 objects misplaced (0.653%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:32:53.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:53 smithi032 ceph-mon[105748]: 1.2 deep-scrub starts 2024-09-21T10:32:53.932 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:53 smithi032 ceph-mon[105748]: 1.2 deep-scrub ok 2024-09-21T10:32:53.946 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:53.979 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:53.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:53.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:54.081 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:32:54.081 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:32:54.081 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:54.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:54.105 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:54.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:54.206 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:54.243 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:54.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:54.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:54.345 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:54.384 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:54.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:54.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:54.486 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:54.521 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:54.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:54.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:54.533 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:32:54.623 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:54.658 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:54.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:54.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:54.760 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:54.800 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:54.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:54.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:54.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:54.933 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:54.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:54.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:54 smithi032 ceph-mon[101299]: osdmap e488: 8 total, 7 up, 7 in 2024-09-21T10:32:54.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:54 smithi032 ceph-mon[105748]: osdmap e488: 8 total, 7 up, 7 in 2024-09-21T10:32:55.034 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:55.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:54 smithi045 ceph-mon[95976]: osdmap e488: 8 total, 7 up, 7 in 2024-09-21T10:32:55.063 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:55.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:55.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:55.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:55.196 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:55.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:55.297 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:55.327 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:55.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:55.406 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:32:55.429 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:55.481 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:55.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:55.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:55.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:55.613 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:55.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:55.714 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:55.760 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:55.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:55.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:55.862 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:55.905 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:55 smithi032 ceph-mon[101299]: pgmap v1499: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 874 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s; 326/41826 objects misplaced (0.779%); 6.6 MiB/s, 106 objects/s recovering 2024-09-21T10:32:55.905 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:55 smithi032 ceph-mon[105748]: pgmap v1499: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 874 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s; 326/41826 objects misplaced (0.779%); 6.6 MiB/s, 106 objects/s recovering 2024-09-21T10:32:55.907 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:55.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:55.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:55.999 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:55 smithi045 ceph-mon[95976]: pgmap v1499: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 874 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s; 326/41826 objects misplaced (0.779%); 6.6 MiB/s, 106 objects/s recovering 2024-09-21T10:32:56.009 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:56.029 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:32:56.030 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:32:56.051 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:56.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:56.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:56.055 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:32:56.056 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:32:56.083 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:32:56.083 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:32:56.108 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:32:56.108 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:32:56.134 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:32:56.134 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:32:56.152 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:56.162 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:32:56.162 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:32:56.196 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:56.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:56.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:56.203 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:32:56.203 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:32:56.231 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:32:56.231 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:32:56.298 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:56.326 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:56.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:56.416 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:32:56.428 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:56.487 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:56.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:56.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:56.589 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:56.619 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:56.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:56.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:56.720 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:56.776 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:56.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:56.878 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:56.914 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/200100097' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:56.919 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:56.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:56.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/200100097' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:56.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/200100097' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:32:57.020 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:57.061 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:57.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:57.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:57.163 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:57.200 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:57.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:57.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:57.302 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:32:57.307 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:32:57.338 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:57.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:57.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:57.440 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:57.473 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:57.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:57.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:57.575 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:57.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:57.606 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:57.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:57.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:57.736 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:57.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:57.838 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:57.880 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:57.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:57.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:57.937 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:57 smithi045 ceph-mon[95976]: pgmap v1500: 109 pgs: 109 active+clean; 798 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:32:57.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:57 smithi032 ceph-mon[101299]: pgmap v1500: 109 pgs: 109 active+clean; 798 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:32:57.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:57 smithi032 ceph-mon[105748]: pgmap v1500: 109 pgs: 109 active+clean; 798 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:32:57.982 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:58.011 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:58.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:58.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:58.113 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:32:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:58.170 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:58.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:32:58.271 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:58.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:58.313 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:58.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:58.414 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:58.458 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:58.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:58.476 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:32:58.476 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:32:58.476 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:32:58.476 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:32:58.477 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:32:58.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:58.588 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:58.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:58.690 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:58.722 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:58.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:58.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:58.823 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:32:58.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:58.862 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:58.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:32:58.963 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:58.967 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4, 1] out_osds: [0] dead_osds: [0] live_osds: [1, 2, 3, 4, 6, 7, 5] 2024-09-21T10:32:58.967 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:32:58.967 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2024-09-21T10:32:58.967 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2024-09-21T10:32:58.967 DEBUG:teuthology.orchestra.run.smithi032:osd.0> sudo journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T10:32:59.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:59.005 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:59.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:59.047 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl start ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:32:59.106 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:32:59.131 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:59.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:59.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:32:59.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:32:59 smithi032 systemd[1]: Starting Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:32:59.226 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:32:59.234 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:59.278 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:59.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:59.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:32:59.308 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:32:59.380 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:32:59.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:59.426 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:59.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:32:59.527 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:59.553 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:32:59.572 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:59.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:59.673 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:32:59.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:32:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:32:59] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.51.0" 2024-09-21T10:32:59.708 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:32:59 smithi032 podman[318874]: 2024-09-21 10:32:59.453489383 +0000 UTC m=+0.047262791 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:32:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:59.721 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:59.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:32:59.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:32:59.841 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is set 2024-09-21T10:32:59.857 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:32:59.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:59.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:32:59.959 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:32:59.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:32:59.999 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:32:59.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:00.014 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:32:59 smithi032 podman[318874]: 2024-09-21 10:32:59.708850319 +0000 UTC m=+0.302623718 container create 635300b11efd116c5a94c1bdbdaa3389753ef29998d10391a61147a1bab9a43d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T10:33:00.014 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:59 smithi032 ceph-mon[105748]: pgmap v1501: 109 pgs: 109 active+clean; 798 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:33:00.014 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3258460129' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:33:00.014 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:32:59 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:33:00.014 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:59 smithi032 ceph-mon[101299]: pgmap v1501: 109 pgs: 109 active+clean; 798 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:33:00.014 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3258460129' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:33:00.015 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:32:59 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:33:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:59 smithi045 ceph-mon[95976]: pgmap v1501: 109 pgs: 109 active+clean; 798 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:33:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3258460129' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:33:00.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:32:59 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:33:00.100 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:00.134 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:00.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:00.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:00.236 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:00.279 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:00.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:00.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:00.381 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:00.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:00.420 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:00.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:00.444 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:00 smithi032 podman[318874]: 2024-09-21 10:33:00.012253681 +0000 UTC m=+0.606027078 container init 635300b11efd116c5a94c1bdbdaa3389753ef29998d10391a61147a1bab9a43d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T10:33:00.444 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:00 smithi032 podman[318874]: 2024-09-21 10:33:00.018509831 +0000 UTC m=+0.612283226 container start 635300b11efd116c5a94c1bdbdaa3389753ef29998d10391a61147a1bab9a43d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:33:00.444 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:00 smithi032 podman[318874]: 2024-09-21 10:33:00.187187122 +0000 UTC m=+0.780960523 container attach 635300b11efd116c5a94c1bdbdaa3389753ef29998d10391a61147a1bab9a43d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T10:33:00.522 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:00.542 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:00.594 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:00.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:00.695 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:00.712 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:00 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[318959]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:33:00.712 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:00 smithi032 bash[318874]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:33:00.712 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:00 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[318959]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:33:00.712 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:00 smithi032 bash[318874]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:33:00.735 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:00.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:00.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:00.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:00.875 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:00.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:00.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:00.976 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:01.002 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:00 smithi032 ceph-mon[105748]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:33:01.003 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:00 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:33:01.003 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:00 smithi032 ceph-mon[105748]: osdmap e489: 8 total, 7 up, 7 in 2024-09-21T10:33:01.003 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:00 smithi032 ceph-mon[101299]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:33:01.003 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:00 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:33:01.003 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:00 smithi032 ceph-mon[101299]: osdmap e489: 8 total, 7 up, 7 in 2024-09-21T10:33:01.015 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:01.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:01.117 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:01.148 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:01.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:01.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:01.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:00 smithi045 ceph-mon[95976]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:33:01.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:00 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:33:01.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:00 smithi045 ceph-mon[95976]: osdmap e489: 8 total, 7 up, 7 in 2024-09-21T10:33:01.249 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[318959]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 bash[318874]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[318959]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 bash[318874]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[318959]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 bash[318874]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[318959]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 bash[318874]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[318959]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 bash[318874]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:33:01.274 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[318959]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:33:01.275 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 bash[318874]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:33:01.275 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate[318959]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T10:33:01.275 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 bash[318874]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T10:33:01.291 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:01.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:01.393 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:01.442 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:01.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:01.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:01.534 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 conmon[318959]: conmon 635300b11efd116c5a94 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-635300b11efd116c5a94c1bdbdaa3389753ef29998d10391a61147a1bab9a43d.scope/container/memory.events 2024-09-21T10:33:01.534 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:01 smithi032 podman[318874]: 2024-09-21 10:33:01.503179519 +0000 UTC m=+2.096952908 container died 635300b11efd116c5a94c1bdbdaa3389753ef29998d10391a61147a1bab9a43d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:33:01.546 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:01.587 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:01.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:01.689 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:01.749 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:01.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:01.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:01.851 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:01.891 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:01.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:01.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:02.009 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:33:02.009 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:33:02.009 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:96597 10.2 10:46baad10:::benchmark_data_smithi032_297285_object96596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:32.143288+0000", 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 149.85783978000001, 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2346571769999999, 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 96597 2024-09-21T10:33:02.010 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.143288+0000", 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.143288+0000", 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.011 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.143287+0000", 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.143326+0000", 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8723000000000002e-05 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.143336+0000", 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.9350000000000006e-06 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.012 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.143342+0000", 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7109999999999998e-06 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.143395+0000", 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2775000000000002e-05 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.143406+0000", 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0682e-05 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.013 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131815+0000", 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98840955200000002 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131824+0000", 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0829999999999993e-06 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.343168+0000", 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.211343482 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.014 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.345285+0000", 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0021170820000000002 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.345645+0000", 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000360415 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.354318+0000", 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0086723649999999996 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.015 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.359702+0000", 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0053847469999999996 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.374486+0000", 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014783895 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.374549+0000", 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2614999999999996e-05 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.016 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.377945+0000", 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0033963610000000001 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:96621 10.2 10:4a6735d3:::benchmark_data_smithi032_297285_object96620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:32.172040+0000", 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 149.82908767500001, 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2096881740000001, 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.017 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 96621 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.172040+0000", 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.172040+0000", 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.018 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.172039+0000", 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.172094+0000", 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5056999999999998e-05 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.172104+0000", 2024-09-21T10:33:02.019 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0345e-05 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.172110+0000", 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2750000000000003e-06 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.172171+0000", 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1162999999999997e-05 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.172181+0000", 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0499999999999999e-05 2024-09-21T10:33:02.020 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131861+0000", 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.95967938500000005 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131867+0000", 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6309999999999997e-06 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.354370+0000", 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2225028099999999 2024-09-21T10:33:02.021 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.354556+0000", 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000185601 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.354747+0000", 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00019091 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.359827+0000", 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0050801259999999999 2024-09-21T10:33:02.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.367138+0000", 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0073115200000000002 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.378007+0000", 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010868361999999999 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.378067+0000", 2024-09-21T10:33:02.023 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0049000000000001e-05 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.381728+0000", 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0036617379999999999 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:96639 10.2 10:4280a108:::benchmark_data_smithi032_297285_object96638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:32.187974+0000", 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 149.81315358699999, 2024-09-21T10:33:02.024 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1973812759999998, 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 96639 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.187974+0000", 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.025 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.187974+0000", 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.187973+0000", 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.188042+0000", 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9119e-05 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.026 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.188063+0000", 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0971e-05 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.188097+0000", 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4233000000000002e-05 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.188174+0000", 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6632999999999996e-05 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.027 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.188184+0000", 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0149999999999999e-05 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131885+0000", 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.94370120700000004 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131891+0000", 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0360000000000003e-06 2024-09-21T10:33:02.028 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.359855+0000", 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2279640620000001 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.360044+0000", 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00018852800000000001 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.360221+0000", 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017732199999999999 2024-09-21T10:33:02.029 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.367179+0000", 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0069573910000000003 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.367216+0000", 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7703999999999997e-05 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.381746+0000", 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014529515999999999 2024-09-21T10:33:02.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.381805+0000", 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9534000000000003e-05 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.385356+0000", 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0035501650000000001 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.031 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:96665 10.2 10:424152fa:::benchmark_data_smithi032_297285_object96664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:32.214667+0000", 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 149.78646138900001, 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1767261489999998, 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 96665 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.032 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.214667+0000", 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.214667+0000", 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.214665+0000", 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.214768+0000", 2024-09-21T10:33:02.033 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000102429 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.214782+0000", 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3876000000000001e-05 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.214789+0000", 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1520000000000003e-06 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.214850+0000", 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0992000000000002e-05 2024-09-21T10:33:02.034 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.035 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.035 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.035 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.214859+0000", 2024-09-21T10:33:02.035 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.3999999999999998e-06 2024-09-21T10:33:02.035 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.035 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131908+0000", 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.91704908600000001 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131914+0000", 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9190000000000004e-06 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.363456+0000", 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2315417710000001 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.036 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.363644+0000", 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000188192 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.363842+0000", 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00019830499999999999 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.377998+0000", 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014155877000000001 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.037 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.385378+0000", 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0073792409999999999 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.389063+0000", 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0036851129999999998 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.389132+0000", 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9065000000000004e-05 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.038 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.391393+0000", 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022608870000000001 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:96671 10.2 10:4bac692e:::benchmark_data_smithi032_297285_object96670:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:32.221952+0000", 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 149.77917583300001, 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1726554550000001, 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.039 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 96671 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.221952+0000", 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.221952+0000", 2024-09-21T10:33:02.040 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.221951+0000", 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222013+0000", 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2762999999999995e-05 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.041 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222029+0000", 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5719999999999999e-05 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222037+0000", 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4760000000000001e-06 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222095+0000", 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8801000000000002e-05 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.042 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222106+0000", 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0890000000000001e-05 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131929+0000", 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.90982305500000005 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131935+0000", 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8390000000000003e-06 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.043 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.367241+0000", 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2353059120000001 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.367435+0000", 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000193909 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.367628+0000", 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00019333499999999999 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.044 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.385391+0000", 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.017762397999999999 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.385449+0000", 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8221999999999999e-05 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.391458+0000", 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.006009113 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.045 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.391501+0000", 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2925e-05 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.394608+0000", 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0031065419999999999 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:96672 10.2 10:4d393ecd:::benchmark_data_smithi032_297285_object96671:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-09-21T10:33:02.046 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:30:32.222048+0000", 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 149.77907943, 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.175357268, 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 96672 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.047 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222048+0000", 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222048+0000", 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222048+0000", 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.048 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222106+0000", 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7975e-05 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222114+0000", 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0560000000000005e-06 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222121+0000", 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1369999999999996e-06 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.049 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222155+0000", 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4140999999999997e-05 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:32.222162+0000", 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1110000000000002e-06 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131953+0000", 2024-09-21T10:33:02.050 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.90979089700000004 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:33.131959+0000", 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0229999999999998e-06 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.370850+0000", 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.238890249 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.371034+0000", 2024-09-21T10:33:02.051 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000184771 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.371235+0000", 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00020055800000000001 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.385397+0000", 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014162587000000001 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.052 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.385490+0000", 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2955999999999996e-05 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.394711+0000", 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0092210750000000005 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.394769+0000", 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7666e-05 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.053 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:30:34.397406+0000", 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002636616 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:122586 10.1 10:8352d8c8:::benchmark_data_smithi032_297285_object5124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:31:06.087924+0000", 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 115.91320392599999, 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0354592629999999, 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:33:02.054 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 122586 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.087924+0000", 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.087924+0000", 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.055 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.087923+0000", 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.087928+0000", 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5659999999999997e-06 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.087944+0000", 2024-09-21T10:33:02.056 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5630000000000001e-05 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.087951+0000", 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.861e-06 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.088004+0000", 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3859000000000001e-05 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.088136+0000", 2024-09-21T10:33:02.057 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013166499999999999 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.088266+0000", 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000130247 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.091715+0000", 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0034489949999999998 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.091939+0000", 2024-09-21T10:33:02.058 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000223926 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.105899+0000", 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013959555 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.105907+0000", 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.388e-06 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.123308+0000", 2024-09-21T10:33:02.059 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.017400986 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.123383+0000", 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5054000000000003e-05 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:122576 10.4 10:25717a1a:::benchmark_data_smithi032_297285_object5114:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:31:06.103497+0000", 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 115.897631092, 2024-09-21T10:33:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0227087640000001, 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 122576 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103497+0000", 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.062 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103497+0000", 2024-09-21T10:33:02.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:02.063 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:02.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:02.063 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.063 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103495+0000", 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103500+0000", 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4479999999999996e-06 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103509+0000", 2024-09-21T10:33:02.064 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.8680000000000005e-06 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103517+0000", 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0930000000000003e-06 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.104449+0000", 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00093209499999999997 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.104457+0000", 2024-09-21T10:33:02.065 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5099999999999998e-06 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.123257+0000", 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.018799464 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.126206+0000", 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029487649999999999 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.066 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:122585 10.4 10:2cd808c2:::benchmark_data_smithi032_297285_object5123:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e455)", 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:31:06.103533+0000", 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 115.897595221, 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0464866129999999, 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 122585 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.067 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103533+0000", 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103533+0000", 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103532+0000", 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.068 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103535+0000", 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8279999999999999e-06 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103541+0000", 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4300000000000003e-06 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.103547+0000", 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9649999999999996e-06 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.069 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.104477+0000", 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00092932300000000002 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:06.104482+0000", 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4859999999999997e-06 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.126694+0000", 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0222120210000001 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.070 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.127618+0000", 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00092429000000000003 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3", 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.128192+0000", 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00057304800000000005 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.142760+0000", 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.01456885 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.071 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.149890+0000", 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0071291849999999997 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.149938+0000", 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8854999999999999e-05 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:31:07.150019+0000", 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0890000000000006e-05 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.072 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:185164 10.4 10:2bfb6abc:::benchmark_data_smithi032_297285_object67702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:04.942832+0000", 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 57.058295844, 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2310840029999999, 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 185164 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.073 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942832+0000", 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942832+0000", 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942831+0000", 2024-09-21T10:33:02.074 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942836+0000", 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8150000000000002e-06 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942848+0000", 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1838000000000001e-05 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942855+0000", 2024-09-21T10:33:02.075 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0679999999999999e-06 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.946160+0000", 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003304478 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.946263+0000", 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000103593 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3", 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.946410+0000", 2024-09-21T10:33:02.076 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014653100000000001 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.969317+0000", 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.022907513000000001 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.978937+0000", 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0096192369999999992 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.978946+0000", 2024-09-21T10:33:02.077 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9050000000000003e-06 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967826+0000", 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.988880859 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967840+0000", 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.3378e-05 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.078 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.173836+0000", 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2059961619999999 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.173916+0000", 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0160999999999994e-05 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:185165 10.1 10:80254b29:::benchmark_data_smithi032_297285_object67703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:04.942863+0000", 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 57.058264821000002, 2024-09-21T10:33:02.079 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2372038910000001, 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 185165 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942863+0000", 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.080 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942863+0000", 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942863+0000", 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942865+0000", 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.311e-06 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.081 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942871+0000", 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5210000000000002e-06 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942875+0000", 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3309999999999997e-06 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942921+0000", 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6458999999999997e-05 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.082 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.943027+0000", 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000105882 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 7", 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.943133+0000", 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000105653 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.949896+0000", 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0067629669999999999 2024-09-21T10:33:02.083 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.975688+0000", 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.025792163999999999 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.975698+0000", 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0104e-05 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967826+0000", 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99212821100000004 2024-09-21T10:33:02.084 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967842+0000", 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.554e-05 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.179996+0000", 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.212153995 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.180067+0000", 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1182999999999999e-05 2024-09-21T10:33:02.085 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:185166 10.4 10:238925ca:::benchmark_data_smithi032_297285_object67704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:04.942887+0000", 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 57.0582414, 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2313324109999999, 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 185166 2024-09-21T10:33:02.086 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942887+0000", 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942887+0000", 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.087 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942886+0000", 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942888+0000", 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.1299999999999999e-06 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942894+0000", 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4319999999999998e-06 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.088 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.942900+0000", 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8629999999999996e-06 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.949914+0000", 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0070144710000000004 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.950038+0000", 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000123838 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.089 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 3", 2024-09-21T10:33:02.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.950156+0000", 2024-09-21T10:33:02.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011864 2024-09-21T10:33:02.090 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.090 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.090 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.090 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.969351+0000", 2024-09-21T10:33:02.090 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.019194378000000002 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.978977+0000", 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0096258219999999992 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.978986+0000", 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9090000000000006e-06 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.091 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967870+0000", 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98888468699999998 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967878+0000", 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0870000000000007e-06 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.174153+0000", 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2062745850000001 2024-09-21T10:33:02.092 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.174219+0000", 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5969999999999993e-05 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:185172 10.4 10:26d87447:::benchmark_data_smithi032_297285_object67710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:04.947556+0000", 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 57.053572148999997, 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2475756580000001, 2024-09-21T10:33:02.093 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 185172 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947556+0000", 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.094 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947556+0000", 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947555+0000", 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947561+0000", 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.2809999999999997e-06 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.095 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.096 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947576+0000", 2024-09-21T10:33:02.096 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5401999999999999e-05 2024-09-21T10:33:02.096 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.096 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.097 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947584+0000", 2024-09-21T10:33:02.097 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.5159999999999997e-06 2024-09-21T10:33:02.097 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.097 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.097 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.979022+0000", 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.031438087000000003 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.979028+0000", 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4160000000000001e-06 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967899+0000", 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.98887083099999995 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.098 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967906+0000", 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3799999999999999e-06 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.174244+0000", 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.206338353 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.174352+0000", 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000107626 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.099 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.180863+0000", 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.006511775 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.194896+0000", 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014032558000000001 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.194935+0000", 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9099e-05 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.100 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.194977+0000", 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1844999999999997e-05 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.195031+0000", 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4582e-05 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.195131+0000", 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000100042 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.101 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:185180 10.4 10:2ef42504:::benchmark_data_smithi032_297285_object67718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:04.977405+0000", 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 57.023722493000001, 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2734814050000001, 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 185180 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.102 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977405+0000", 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977405+0000", 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977404+0000", 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.103 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977410+0000", 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5570000000000003e-06 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977419+0000", 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6690000000000001e-06 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977428+0000", 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6689999999999995e-06 2024-09-21T10:33:02.104 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.979053+0000", 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001625179 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.979060+0000", 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.1579999999999999e-06 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967925+0000", 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.988864403 2024-09-21T10:33:02.105 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967931+0000", 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0499999999999997e-06 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.188330+0000", 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2203997209999999 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.188476+0000", 2024-09-21T10:33:02.106 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014535500000000001 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.188619+0000", 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014313100000000001 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.194943+0000", 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0063239500000000001 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.195177+0000", 2024-09-21T10:33:02.107 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000234467 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.250683+0000", 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.055505947 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.250745+0000", 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1416999999999998e-05 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.108 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.250887+0000", 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014220899999999999 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:185181 10.1 10:8aecb090:::benchmark_data_smithi032_297285_object67719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:04.977436+0000", 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 57.023692285999999, 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.273509496, 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.109 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 185181 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977436+0000", 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977436+0000", 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.110 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977435+0000", 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977438+0000", 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.8430000000000001e-06 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977445+0000", 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8809999999999998e-06 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.111 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977450+0000", 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5090000000000001e-06 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977597+0000", 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014671399999999999 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.977605+0000", 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9640000000000003e-06 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.112 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967869+0000", 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99026349800000002 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.967876+0000", 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2139999999999999e-06 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.181472+0000", 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2135962440000001 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.113 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.181586+0000", 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000113838 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.182099+0000", 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00051334700000000004 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.192406+0000", 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010307273 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.114 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.204067+0000", 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011660245 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.250846+0000", 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.046779464 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.250877+0000", 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0378999999999998e-05 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.115 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.250945+0000", 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8580999999999999e-05 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:208441 10.1 10:8b5511ca:::benchmark_data_smithi032_297285_object90979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:36.124262+0000", 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 25.876866299, 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.89363378199999999, 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.116 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 208441 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.124262+0000", 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.117 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.118 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:01 smithi045 ceph-mon[95976]: pgmap v1503: 109 pgs: 109 active+clean; 514 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 2.4 MiB/s, 37 objects/s recovering 2024-09-21T10:33:02.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.124262+0000", 2024-09-21T10:33:02.118 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.118 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.118 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.118 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.118 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.124260+0000", 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.124265+0000", 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4149999999999998e-06 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.124275+0000", 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.5599999999999999e-06 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.119 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.124281+0000", 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4459999999999998e-06 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.125388+0000", 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0011072180000000001 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.125510+0000", 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000121935 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.120 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.125632+0000", 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000122063 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.134418+0000", 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087852380000000008 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.134427+0000", 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4150000000000005e-06 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.121 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.017816+0000", 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.88338947999999995 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.017895+0000", 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8977e-05 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:208488 10.1 10:8e3d8c07:::benchmark_data_smithi032_297285_object91026:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2024-09-21T10:33:02.122 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:36.172364+0000", 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 25.828764100000001, 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.851999375, 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 208488 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.172364+0000", 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.123 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.172364+0000", 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.172362+0000", 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.172369+0000", 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9450000000000004e-06 2024-09-21T10:33:02.124 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.172389+0000", 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9573e-05 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.172396+0000", 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3629999999999996e-06 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.172453+0000", 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.7012999999999998e-05 2024-09-21T10:33:02.125 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.172461+0000", 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1380000000000007e-06 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.018221+0000", 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.84575986000000003 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.018321+0000", 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.9951000000000006e-05 2024-09-21T10:33:02.126 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.018404+0000", 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2992999999999996e-05 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.024185+0000", 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0057813300000000003 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.024234+0000", 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8631999999999998e-05 2024-09-21T10:33:02.127 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.024273+0000", 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8988e-05 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.024322+0000", 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9570999999999997e-05 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.024363+0000", 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0768e-05 2024-09-21T10:33:02.128 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:208495 10.1 10:8f9ce365:::benchmark_data_smithi032_297285_object91033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:36.178774+0000", 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 25.822353723999999, 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.85137169099999999, 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 208495 2024-09-21T10:33:02.129 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.178774+0000", 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.178774+0000", 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.178772+0000", 2024-09-21T10:33:02.130 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.178780+0000", 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.588e-06 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.178796+0000", 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6171000000000002e-05 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.178803+0000", 2024-09-21T10:33:02.131 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3250000000000002e-06 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.180715+0000", 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.001911427 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.180723+0000", 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.6470000000000004e-06 2024-09-21T10:33:02.132 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.021134+0000", 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.84041032599999999 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.021255+0000", 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000121458 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.021402+0000", 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000146599 2024-09-21T10:33:02.133 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.024450+0000", 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00304792 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.027322+0000", 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0028725320000000001 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030065+0000", 2024-09-21T10:33:02.134 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0027426400000000002 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030108+0000", 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3467999999999998e-05 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030146+0000", 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7460999999999999e-05 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.135 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:208508 10.1 10:88c38cb2:::benchmark_data_smithi032_297285_object91046:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:36.190394+0000", 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 25.8107337, 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.83995207299999997, 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 208508 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.136 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.190394+0000", 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.190394+0000", 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.190392+0000", 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.137 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.190400+0000", 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6809999999999998e-06 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.190417+0000", 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6957e-05 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.190425+0000", 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.1629999999999996e-06 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.138 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.190473+0000", 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8126e-05 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.190482+0000", 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.9400000000000008e-06 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.024499+0000", 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.83401647400000001 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.139 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.024597+0000", 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.7777999999999998e-05 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.024711+0000", 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011397599999999999 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030204+0000", 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0054933489999999998 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030240+0000", 2024-09-21T10:33:02.140 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.5732999999999998e-05 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030248+0000", 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8849999999999999e-06 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030291+0000", 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.3090000000000002e-05 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030346+0000", 2024-09-21T10:33:02.141 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5665999999999999e-05 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:208525 10.1 10:8bd70489:::benchmark_data_smithi032_297285_object91063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:36.208135+0000", 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 25.792992591000001, 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.82803673200000005, 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:02.142 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 208525 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.208135+0000", 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.208135+0000", 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:02.143 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.208134+0000", 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.208139+0000", 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6890000000000003e-06 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.208150+0000", 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1469e-05 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:02.144 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.208158+0000", 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.0819999999999999e-06 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.208647+0000", 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00048859100000000003 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:36.208656+0000", 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.4099999999999997e-06 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:02.145 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.027368+0000", 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.818711886 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.027496+0000", 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000127918 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.027633+0000", 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013706999999999999 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:02.146 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030355+0000", 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0027221390000000002 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.030406+0000", 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0608000000000001e-05 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.036096+0000", 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0056900859999999996 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:02.147 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.036138+0000", 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1915000000000001e-05 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:37.036172+0000", 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4270000000000002e-05 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:02.148 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:33:02.164 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:02.202 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:02.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:02.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:01 smithi032 ceph-mon[105748]: pgmap v1503: 109 pgs: 109 active+clean; 514 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 2.4 MiB/s, 37 objects/s recovering 2024-09-21T10:33:02.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:01 smithi032 ceph-mon[101299]: pgmap v1503: 109 pgs: 109 active+clean; 514 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 2.4 MiB/s, 37 objects/s recovering 2024-09-21T10:33:02.304 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:02.343 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:02.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:02.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:02.445 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:02.483 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:02.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:02.585 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:02.630 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:02.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:02.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:02 smithi032 podman[319265]: 2024-09-21 10:33:02.437125349 +0000 UTC m=+1.155074646 container remove 635300b11efd116c5a94c1bdbdaa3389753ef29998d10391a61147a1bab9a43d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-activate, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T10:33:02.927 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:02.960 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:02.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:02.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:03.062 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:03.101 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:03.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:03.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:03.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:33:03.203 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:03.237 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:03.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:03.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:03.267 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:33:03.339 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:03.399 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:03.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:03.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:03.501 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:03.553 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:03.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:03.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:03.654 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:03.706 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:03.709 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:33:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:03.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:03.738 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:03 smithi032 podman[319402]: 2024-09-21 10:33:03.394553897 +0000 UTC m=+0.031824277 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:33:03.810 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:03.841 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:03.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:03.920 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:03 smithi045 ceph-mon[95976]: pgmap v1504: 109 pgs: 109 active+clean; 473 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 990 op/s; 2.0 MiB/s, 32 objects/s recovering 2024-09-21T10:33:03.920 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:03.943 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:03.977 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:03.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:04.015 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:03 smithi032 ceph-mon[105748]: pgmap v1504: 109 pgs: 109 active+clean; 473 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 990 op/s; 2.0 MiB/s, 32 objects/s recovering 2024-09-21T10:33:04.015 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:04.016 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:03 smithi032 ceph-mon[101299]: pgmap v1504: 109 pgs: 109 active+clean; 473 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 990 op/s; 2.0 MiB/s, 32 objects/s recovering 2024-09-21T10:33:04.016 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:04.079 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:04.108 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:04.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:04.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:04.210 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:04.253 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:04.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:04.269 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:04 smithi032 podman[319402]: 2024-09-21 10:33:04.013778001 +0000 UTC m=+0.651048377 container create f88bf0350cada02bc5dc6933b8ee1f54abfde15f8fbab526fff2f1c832c166a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-21T10:33:04.355 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:04.391 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:04.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:04.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:04.398 INFO:teuthology.orchestra.run.smithi032.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:33:04.429 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:33:04.429 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:33:04.465 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:33:04.466 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:33:04.493 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:04.497 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:33:04.497 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:33:04.529 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:33:04.529 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:33:04.529 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:04 smithi032 podman[319402]: 2024-09-21 10:33:04.510681095 +0000 UTC m=+1.147951477 container init f88bf0350cada02bc5dc6933b8ee1f54abfde15f8fbab526fff2f1c832c166a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:33:04.529 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:04 smithi032 podman[319402]: 2024-09-21 10:33:04.516987755 +0000 UTC m=+1.154258128 container start f88bf0350cada02bc5dc6933b8ee1f54abfde15f8fbab526fff2f1c832c166a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T10:33:04.538 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:04.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:04.565 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:33:04.566 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:33:04.593 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:33:04.593 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:33:04.626 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:33:04.626 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:33:04.636 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:33:04.640 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:04.674 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:04.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:04.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:04.775 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:04.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:04.807 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:04.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:04.871 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:04 smithi032 ceph-osd[319551]: -- 172.21.15.32:0/1247341301 <== mon.2 v2:172.21.15.32:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x55ce3d5f41a0 con 0x55ce3d5c2400 2024-09-21T10:33:04.871 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:04 smithi032 ceph-osd[319551]: -- 172.21.15.32:0/1247341301 <== mon.2 v2:172.21.15.32:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ce3c7f5a40 con 0x55ce3d5c2400 2024-09-21T10:33:04.871 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:04 smithi032 bash[319402]: f88bf0350cada02bc5dc6933b8ee1f54abfde15f8fbab526fff2f1c832c166a9 2024-09-21T10:33:04.871 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:04 smithi032 systemd[1]: Started Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:33:04.909 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:04.949 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:04.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:04.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:05.051 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:05.082 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:05.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:05.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:05.158 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:04 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4288028805' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:33:05.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:04 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4288028805' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:33:05.184 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:05.213 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:05.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:05.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:05.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:04 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4288028805' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:33:05.314 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:05.343 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:05.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:05.446 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:05.489 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:05.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:05.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:05.591 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:05.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:05.629 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:05.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:05.730 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:05.765 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:05.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:05.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:05.866 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:05.892 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:05 smithi045 ceph-mon[95976]: pgmap v1505: 109 pgs: 109 active+clean; 256 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:33:05.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:05.905 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:05.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:05.985 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:33:05.985 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:33:06.006 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:06.035 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:06.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:06.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:06.136 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:06.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:06.167 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:06.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:06.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:05 smithi032 ceph-mon[105748]: pgmap v1505: 109 pgs: 109 active+clean; 256 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:33:06.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:05 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[319501]: 2024-09-21T10:33:05.902+0000 7f8771e85640 -1 Falling back to public interface 2024-09-21T10:33:06.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:05 smithi032 ceph-mon[101299]: pgmap v1505: 109 pgs: 109 active+clean; 256 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:33:06.269 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:06.312 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:06.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:06.414 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:06.454 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:06.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:06.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:06.555 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:06.590 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:06.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:06.688 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:33:06.690 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:06.775 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:06.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:06.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:06.877 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:06.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:06.924 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:06.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:07.025 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:07.069 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:33:07.075 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:07.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:07.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:07.177 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:07.213 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:07.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:07.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:07.233 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Removed 117459 objects 2024-09-21T10:33:07.233 INFO:tasks.radosbench.radosbench.0.smithi032.stdout:Clean up completed and total clean up time :126.034 2024-09-21T10:33:07.315 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:07.337 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:07.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:07.439 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:07.463 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:07.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:07.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:07.564 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:33:07.585 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:33:07.587 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:07.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:07.658 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:07.688 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:07.710 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:07.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:07.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:07.812 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:07.834 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:07.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:07.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:07.933 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:07.935 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:07.957 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:07.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:08.059 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:08.081 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:08.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:08.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:08.140 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:07 smithi032 ceph-mon[101299]: pgmap v1506: 109 pgs: 109 active+clean; 156 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:33:08.140 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:07 smithi032 ceph-mon[105748]: pgmap v1506: 109 pgs: 109 active+clean; 156 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:33:08.184 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:08.205 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:08.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:08.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:07 smithi045 ceph-mon[95976]: pgmap v1506: 109 pgs: 109 active+clean; 156 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:33:08.307 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:08.330 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:08.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:08.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:08.431 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:08.454 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:08.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:08.555 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:08.579 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:08.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:08.681 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:08.717 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:08.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:08.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:08.819 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:08.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:08.846 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:08.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:08.880 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:33:08.880 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:33:08.948 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:08.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:08.972 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:08.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:09.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:09.094 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:09.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:09.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:09.196 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:09.219 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:09.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:09.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:09.321 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:09.343 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:09.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:09.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:09.444 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:09.466 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:09.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:09.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:09.524 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:33:09.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:09.586 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:33:09.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:09.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:09.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:09.692 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:09.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:09.720 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:09.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:09.729 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:33:09.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:09.843 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:09.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:09.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:09.894 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:33:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:33:09] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.51.0" 2024-09-21T10:33:09.945 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:09.967 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:09.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:10.069 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:10.091 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:10.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:10.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:10.114 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:09 smithi045 ceph-mon[95976]: pgmap v1507: 109 pgs: 109 active+clean; 156 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:33:10.190 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:09 smithi032 ceph-mon[101299]: pgmap v1507: 109 pgs: 109 active+clean; 156 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:33:10.191 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:09 smithi032 ceph-mon[105748]: pgmap v1507: 109 pgs: 109 active+clean; 156 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:33:10.193 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:10.216 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:10.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:10.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:10.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:10.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:10.341 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:10.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:10.442 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:10.463 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:10.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:10.565 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:10.587 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:10.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:10.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:10.689 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:10.711 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:10.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:10.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:10.751 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-09-21T10:33:10.752 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-09-21T10:33:10.813 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:10.835 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:10.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:10.937 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:10.960 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:10.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:10.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:11.031 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:10 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:33:11.032 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:10 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:33:11.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:11.084 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:11.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:11.164 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:10 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:33:11.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:11.210 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:11.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:11.312 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:11.318 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[319501]: 2024-09-21T10:33:11.133+0000 7f8771e85640 -1 osd.0 454 log_to_monitors true 2024-09-21T10:33:11.335 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:11.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:11.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:11.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:11.459 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:11.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:11.560 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:11.583 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:11.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:11.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:11.609 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:33:11.609 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:33:11.685 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:11.707 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:11.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:11.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:no valid command found; 10 closest matches: 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:0 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:1 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:2 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:abort 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:assert 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:bluefs files list 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:bluefs stats 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:bluestore allocator dump block 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:bluestore allocator fragmentation block 2024-09-21T10:33:11.743 INFO:teuthology.orchestra.run.smithi032.stderr:admin_socket: invalid command 2024-09-21T10:33:11.809 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:11.831 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:11.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:11.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:11.909 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is set 2024-09-21T10:33:11.933 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:11.956 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:11.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:11.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:12.058 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:12.081 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:12.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:12.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:12.183 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:12.206 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:12.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:12.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:12.207 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[319501]: 2024-09-21T10:33:11.925+0000 7f8765c27640 -1 osd.0 454 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:33:12.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-mon[101299]: pgmap v1508: 109 pgs: 109 active+clean; 11 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 823 op/s 2024-09-21T10:33:12.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-mon[101299]: from='osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:33:12.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2046866495' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:12.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:12.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-mon[105748]: pgmap v1508: 109 pgs: 109 active+clean; 11 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 823 op/s 2024-09-21T10:33:12.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-mon[105748]: from='osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:33:12.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2046866495' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:12.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:11 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:11 smithi045 ceph-mon[95976]: pgmap v1508: 109 pgs: 109 active+clean; 11 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 823 op/s 2024-09-21T10:33:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:11 smithi045 ceph-mon[95976]: from='osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:33:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:11 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2046866495' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:11 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:12.308 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:12.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:33:12.334 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:12.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:12.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:12.436 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:12.458 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:12.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:12.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:12.560 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:12.585 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:12.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:12.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:12.608 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:33:12.609 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-0, ['dump_ops_in_flight'] 2024-09-21T10:33:12.686 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:12.712 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:12.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:12.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:12.814 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:12.837 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:12.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:12.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:12.938 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:12.973 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:12.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:12.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:13.065 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:33:13.075 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:13.100 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:13.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:13.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:13.190 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:12 smithi045 ceph-mon[95976]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:33:13.190 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:12 smithi045 ceph-mon[95976]: from='osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:33:13.190 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:12 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:33:13.190 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:12 smithi045 ceph-mon[95976]: osdmap e490: 8 total, 7 up, 7 in 2024-09-21T10:33:13.190 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:12 smithi045 ceph-mon[95976]: from='osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:33:13.203 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[105748]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:33:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[105748]: from='osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:33:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:33:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[105748]: osdmap e490: 8 total, 7 up, 7 in 2024-09-21T10:33:13.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[105748]: from='osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:33:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[101299]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:33:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[101299]: from='osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:33:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:33:13.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[101299]: osdmap e490: 8 total, 7 up, 7 in 2024-09-21T10:33:13.208 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:12 smithi032 ceph-mon[101299]: from='osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]} : dispatch 2024-09-21T10:33:13.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:13.225 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:13.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:13.326 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:13.350 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:13.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:13.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:13.452 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:13.474 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:13.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:13.576 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:13.599 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:13.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:13.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:13.694 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:13.701 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:13.726 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:13.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:13.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:13.743 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:13.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:13.851 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:13.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:13.952 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:13 smithi032 ceph-mon[101299]: pgmap v1510: 109 pgs: 109 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 748 op/s 2024-09-21T10:33:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:13 smithi032 ceph-mon[101299]: osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695] boot 2024-09-21T10:33:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:13 smithi032 ceph-mon[101299]: osdmap e491: 8 total, 8 up, 7 in 2024-09-21T10:33:13.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:13 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:33:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:13 smithi032 ceph-mon[105748]: pgmap v1510: 109 pgs: 109 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 748 op/s 2024-09-21T10:33:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:13 smithi032 ceph-mon[105748]: osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695] boot 2024-09-21T10:33:13.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:13 smithi032 ceph-mon[105748]: osdmap e491: 8 total, 8 up, 7 in 2024-09-21T10:33:13.958 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:13 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:33:13.975 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:13.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:13.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:14.076 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:14.098 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:14.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:14.157 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:13 smithi045 ceph-mon[95976]: pgmap v1510: 109 pgs: 109 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 748 op/s 2024-09-21T10:33:14.157 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:13 smithi045 ceph-mon[95976]: osd.0 [v2:172.21.15.32:6802/2741331695,v1:172.21.15.32:6803/2741331695] boot 2024-09-21T10:33:14.157 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:13 smithi045 ceph-mon[95976]: osdmap e491: 8 total, 8 up, 7 in 2024-09-21T10:33:14.157 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:13 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:33:14.200 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:14.223 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:14.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:14.324 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:14.346 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:14.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:14.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:14.448 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:14.470 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:14.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:14.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:14.572 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:14.594 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:14.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:14.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:14.696 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:14.718 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:14.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:14.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:14.820 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:14.842 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:14.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:14.872 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:33:14.873 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:33:14.944 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:14.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:14.967 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:14.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:15.068 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:15.090 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:15.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:15.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:15.191 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:15.215 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:15.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:15.316 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:15.339 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:15.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:15.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:15.441 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:15.464 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:15.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:15.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:15.532 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:33:15.566 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:15.590 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:15.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:15.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:15.692 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:15.715 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:15.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:15.716 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:33:15.817 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:15.839 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:15.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:15.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:15.940 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:15.962 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:15.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:16.064 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:16.075 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:33:16.087 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:16.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:16.188 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:16.198 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:15 smithi045 ceph-mon[95976]: pgmap v1512: 109 pgs: 109 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 302 op/s 2024-09-21T10:33:16.206 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:15 smithi032 ceph-mon[105748]: pgmap v1512: 109 pgs: 109 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 302 op/s 2024-09-21T10:33:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:15 smithi032 ceph-mon[101299]: pgmap v1512: 109 pgs: 109 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 302 op/s 2024-09-21T10:33:16.210 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:16.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:16.312 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:16.333 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:16.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:16.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:16.435 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:16.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:16.459 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:16.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:16.560 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:16.583 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:16.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:16.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:16.685 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:16.706 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:16.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:16.808 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:16.833 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:16.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:16.916 INFO:teuthology.orchestra.run.smithi032.stderr:pool 'unique_pool_5' removed 2024-09-21T10:33:16.935 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:16.979 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:16.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:17.065 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:16 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/824344005' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:33:17.065 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:16 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/824344005' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:33:17.081 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:17.103 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:17.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:17.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:17.205 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:17.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:17.228 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:17.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:17.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:16 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/824344005' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:33:17.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:17.352 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:17.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:17.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:17.453 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:17.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:17.478 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:17.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:17.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:17.603 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:17.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:17.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:17.610 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:33:17.704 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:17.708 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T10:33:17.708 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-21T10:33:17.708 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-21T10:33:17.709 INFO:tasks.workunit:Pulling workunits from ref reef 2024-09-21T10:33:17.709 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-21T10:33:17.709 DEBUG:teuthology.orchestra.run.smithi032:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-21T10:33:17.728 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:17.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:17.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:17.735 INFO:teuthology.orchestra.run.smithi032.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-21T10:33:17.736 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T10:33:17.800 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-21T10:33:17.800 DEBUG:teuthology.orchestra.run.smithi032:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-21T10:33:17.829 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:17.856 INFO:tasks.workunit:timeout=3h 2024-09-21T10:33:17.856 INFO:tasks.workunit:cleanup=True 2024-09-21T10:33:17.857 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-21T10:33:17.864 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:17.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:17.890 INFO:tasks.workunit.client.0.smithi032.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-21T10:33:17.956 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:33:17.957 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:33:17.957 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-21T10:33:17.957 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:33:17.965 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:17.972 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:33:17.972 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:33:17.990 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:17.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:17.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:17.991 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:33:17.991 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:33:18.007 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:33:18.007 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:33:18.023 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:33:18.023 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:33:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:17 smithi045 ceph-mon[95976]: pgmap v1513: 109 pgs: 109 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 302 op/s 2024-09-21T10:33:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:17 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/824344005' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:33:18.035 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:17 smithi045 ceph-mon[95976]: osdmap e492: 8 total, 8 up, 7 in 2024-09-21T10:33:18.039 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:33:18.040 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:33:18.056 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:33:18.056 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:33:18.091 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:18.113 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:18.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:18.195 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:17 smithi032 ceph-mon[105748]: pgmap v1513: 109 pgs: 109 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 302 op/s 2024-09-21T10:33:18.195 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:17 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/824344005' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:33:18.195 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:17 smithi032 ceph-mon[105748]: osdmap e492: 8 total, 8 up, 7 in 2024-09-21T10:33:18.195 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:17 smithi032 ceph-mon[101299]: pgmap v1513: 109 pgs: 109 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 302 op/s 2024-09-21T10:33:18.195 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:17 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/824344005' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:33:18.195 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:17 smithi032 ceph-mon[101299]: osdmap e492: 8 total, 8 up, 7 in 2024-09-21T10:33:18.215 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:18.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:18.242 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:18.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:18.343 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:18.365 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:18.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:18.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:18.467 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:18.489 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:18.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:18.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:18.591 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:18.613 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:18.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:18.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:18.638 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:33:18.638 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:33:18.638 INFO:teuthology.orchestra.run.smithi045.stdout: "complaint_time": 30, 2024-09-21T10:33:18.638 INFO:teuthology.orchestra.run.smithi045.stdout: "num_blocked_ops": 0 2024-09-21T10:33:18.638 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:33:18.715 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:18.737 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:18.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:18.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:18.839 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:18.862 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:18.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:18.964 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:18.986 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:18.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:19.088 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:19.111 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:19.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:19.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:18 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2213011078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:33:19.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:18 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2213011078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:33:19.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:19.213 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:19.235 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:19.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:19.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:19.259 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:18 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2213011078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:33:19.259 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:19.281 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:33:19.337 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:19.359 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:19.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:19.460 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:19.483 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:19.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:19.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:19.494 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:19.522 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:19.585 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:19.608 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:19.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:19.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:19.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:19.736 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:19.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:19.807 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:33:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:33:19] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-09-21T10:33:19.837 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:19.859 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:19.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:19.961 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:19.983 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:19.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:19.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:20.085 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:20.108 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:20.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:20.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:20.123 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:19 smithi032 ceph-mon[101299]: pgmap v1515: 93 pgs: 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:20.123 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:19 smithi032 ceph-mon[105748]: pgmap v1515: 93 pgs: 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:20.209 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:20.231 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:20.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:20.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:20.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:19 smithi045 ceph-mon[95976]: pgmap v1515: 93 pgs: 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:20.333 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:20.356 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:20.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:20.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:20.457 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:20.481 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:20.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:20.583 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:20.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:20.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:20.707 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:20.730 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:20.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:20.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:20.832 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:20.856 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:20.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:20.928 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-21T10:33:20.929 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]} : dispatch 2024-09-21T10:33:20.929 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 7]} : dispatch 2024-09-21T10:33:20.929 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:20 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 7]} : dispatch 2024-09-21T10:33:20.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:20.980 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:20.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:20.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:21.022 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:33:21.082 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:21.106 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:21.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:21.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-21T10:33:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]} : dispatch 2024-09-21T10:33:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 7]} : dispatch 2024-09-21T10:33:21.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:20 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 7]} : dispatch 2024-09-21T10:33:21.207 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:21.229 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:21.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:21.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:21.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-21T10:33:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]} : dispatch 2024-09-21T10:33:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 7]} : dispatch 2024-09-21T10:33:21.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:20 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 7]} : dispatch 2024-09-21T10:33:21.331 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:21.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:21.358 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:21.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:21.459 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:21.481 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:21.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:21.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:21.582 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:21.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:21.605 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:21.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:21.706 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:21.732 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:21.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:21.766 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:33:21.834 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:21.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:21.859 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:21.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:21.960 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:21.982 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:21.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:21.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:22.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:22.106 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:22.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:22.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:22.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[101299]: pgmap v1516: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:22.159 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 7]}]': finished 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 7]}]': finished 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[101299]: osdmap e493: 8 total, 8 up, 7 in 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[105748]: pgmap v1516: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 7]}]': finished 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 7]}]': finished 2024-09-21T10:33:22.160 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[105748]: osdmap e493: 8 total, 8 up, 7 in 2024-09-21T10:33:22.161 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:22.161 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:22.173 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:21 smithi045 ceph-mon[95976]: pgmap v1516: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:22.174 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-21T10:33:22.174 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-21T10:33:22.174 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 7]}]': finished 2024-09-21T10:33:22.174 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 7]}]': finished 2024-09-21T10:33:22.174 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:21 smithi045 ceph-mon[95976]: osdmap e493: 8 total, 8 up, 7 in 2024-09-21T10:33:22.174 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:22.174 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:22.208 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:22.232 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:22.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:22.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:22.334 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:22.356 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:22.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:22.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:22.458 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:22.480 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:22.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:22.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:22.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:22.605 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:22.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:22.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:22.707 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:22.730 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:22.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:22.832 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:22.854 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:22.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:22.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:22.956 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:22.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:22 smithi032 ceph-mon[105748]: osdmap e494: 8 total, 8 up, 7 in 2024-09-21T10:33:22.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:22 smithi032 ceph-mon[101299]: osdmap e494: 8 total, 8 up, 7 in 2024-09-21T10:33:22.980 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:22.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:22.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:23.081 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:23.104 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:23.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:23.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:23.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:23.228 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:23.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:23.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:23.250 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:22 smithi045 ceph-mon[95976]: osdmap e494: 8 total, 8 up, 7 in 2024-09-21T10:33:23.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:23.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:23.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:23.454 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:23.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:23.478 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:23.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:23.494 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:33:23.495 INFO:teuthology.orchestra.run.smithi045.stdout: "size": 20, 2024-09-21T10:33:23.495 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 600, 2024-09-21T10:33:23.495 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [ 2024-09-21T10:33:23.495 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.495 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:142730 9.3 9:caf5744d:::benchmark_data_smithi032_265250_object14928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-09-21T10:33:23.496 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.656116+0000", 2024-09-21T10:33:23.496 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 420.83510262099998, 2024-09-21T10:33:23.496 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14369104699999999, 2024-09-21T10:33:23.496 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.496 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.496 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.497 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:33:23.497 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:33:23.497 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 142730 2024-09-21T10:33:23.497 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.497 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.497 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.497 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.497 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656116+0000", 2024-09-21T10:33:23.497 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.498 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.498 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.498 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.498 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656116+0000", 2024-09-21T10:33:23.498 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.498 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.498 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.498 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.498 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656115+0000", 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656120+0000", 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.6609999999999997e-06 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656131+0000", 2024-09-21T10:33:23.499 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0849000000000001e-05 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656138+0000", 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9199999999999998e-06 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656179+0000", 2024-09-21T10:33:23.500 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1362000000000001e-05 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656278+0000", 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.8797999999999998e-05 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.656398+0000", 2024-09-21T10:33:23.501 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120029 2024-09-21T10:33:23.502 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.502 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.502 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.502 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.659952+0000", 2024-09-21T10:33:23.502 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0035538010000000001 2024-09-21T10:33:23.502 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.502 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.502 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.502 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797529+0000", 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13757687900000001 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799595+0000", 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0020662549999999999 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.503 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799761+0000", 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016636099999999999 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799807+0000", 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5318000000000002e-05 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.504 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:142731 9.3 9:cf90c21c:::benchmark_data_smithi032_265250_object14929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.660201+0000", 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 420.83101689699998, 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13972269700000001, 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:33:23.505 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:33:23.506 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 142731 2024-09-21T10:33:23.506 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.506 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.506 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.506 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.506 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660201+0000", 2024-09-21T10:33:23.506 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.506 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.506 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660201+0000", 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660199+0000", 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.507 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.508 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.508 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660206+0000", 2024-09-21T10:33:23.508 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7139999999999996e-06 2024-09-21T10:33:23.508 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.508 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.508 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.508 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660218+0000", 2024-09-21T10:33:23.508 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1960000000000001e-05 2024-09-21T10:33:23.508 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.509 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.509 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.509 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660225+0000", 2024-09-21T10:33:23.509 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7619999999999998e-06 2024-09-21T10:33:23.509 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.509 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.510 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.510 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660255+0000", 2024-09-21T10:33:23.510 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.0051000000000001e-05 2024-09-21T10:33:23.510 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.510 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.510 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.511 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660328+0000", 2024-09-21T10:33:23.511 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.3061999999999999e-05 2024-09-21T10:33:23.511 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.511 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.511 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:33:23.511 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.660405+0000", 2024-09-21T10:33:23.512 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6862999999999999e-05 2024-09-21T10:33:23.512 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.512 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.512 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.512 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663603+0000", 2024-09-21T10:33:23.512 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003198116 2024-09-21T10:33:23.512 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.513 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.513 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.513 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797574+0000", 2024-09-21T10:33:23.513 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.133970802 2024-09-21T10:33:23.513 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.513 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.513 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.514 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799849+0000", 2024-09-21T10:33:23.514 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022749430000000002 2024-09-21T10:33:23.514 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.514 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.514 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.514 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799888+0000", 2024-09-21T10:33:23.515 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.8939000000000001e-05 2024-09-21T10:33:23.515 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.515 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.515 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.515 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799924+0000", 2024-09-21T10:33:23.515 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.6594999999999998e-05 2024-09-21T10:33:23.516 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.516 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.516 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.516 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.516 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.516 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:142733 9.d 9:bb8717e3:::benchmark_data_smithi032_265250_object14931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-09-21T10:33:23.517 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.663640+0000", 2024-09-21T10:33:23.517 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 420.82757849299998, 2024-09-21T10:33:23.517 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14898618299999999, 2024-09-21T10:33:23.517 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.517 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.517 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.517 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:33:23.518 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:33:23.518 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 142733 2024-09-21T10:33:23.518 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.518 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.518 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.518 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.519 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663640+0000", 2024-09-21T10:33:23.519 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.519 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.519 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.519 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.519 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663640+0000", 2024-09-21T10:33:23.520 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.520 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.520 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.520 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.520 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663639+0000", 2024-09-21T10:33:23.520 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:23.520 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.521 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.521 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.521 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663644+0000", 2024-09-21T10:33:23.521 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4620000000000004e-06 2024-09-21T10:33:23.521 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.522 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.522 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.522 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663653+0000", 2024-09-21T10:33:23.522 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.0729999999999994e-06 2024-09-21T10:33:23.522 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.522 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.523 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.523 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663659+0000", 2024-09-21T10:33:23.523 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1630000000000001e-06 2024-09-21T10:33:23.523 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.523 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.523 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.524 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663691+0000", 2024-09-21T10:33:23.524 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1903000000000003e-05 2024-09-21T10:33:23.524 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.524 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.524 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.524 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.663789+0000", 2024-09-21T10:33:23.525 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.7422000000000003e-05 2024-09-21T10:33:23.525 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.525 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.525 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:33:23.525 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797500+0000", 2024-09-21T10:33:23.525 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13371169299999999 2024-09-21T10:33:23.526 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.526 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.526 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.526 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.811231+0000", 2024-09-21T10:33:23.526 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.013731220000000001 2024-09-21T10:33:23.526 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.527 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.527 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.527 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.811287+0000", 2024-09-21T10:33:23.527 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5899999999999997e-05 2024-09-21T10:33:23.527 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.527 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.528 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.528 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.812552+0000", 2024-09-21T10:33:23.528 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0012642 2024-09-21T10:33:23.528 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.528 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.529 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.529 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.812593+0000", 2024-09-21T10:33:23.529 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1242999999999998e-05 2024-09-21T10:33:23.529 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.529 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.529 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.530 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.812626+0000", 2024-09-21T10:33:23.530 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3198999999999998e-05 2024-09-21T10:33:23.530 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.530 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.530 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.530 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.530 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.531 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop_reply(client.45848.0:142730 9.3 e394/378 ondisk, result = 0)", 2024-09-21T10:33:23.531 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:22.665234+0000", 2024-09-21T10:33:23.531 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 420.82598423799999, 2024-09-21T10:33:23.531 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13458094600000001, 2024-09-21T10:33:23.531 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.531 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "started", 2024-09-21T10:33:23.531 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.531 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.532 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.532 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.665234+0000", 2024-09-21T10:33:23.532 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.532 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.532 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.532 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.532 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.665234+0000", 2024-09-21T10:33:23.533 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.533 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.533 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.533 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.533 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.665233+0000", 2024-09-21T10:33:23.533 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:33:23.534 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.534 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.534 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.534 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.665239+0000", 2024-09-21T10:33:23.534 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.1770000000000003e-06 2024-09-21T10:33:23.534 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.535 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.535 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.535 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797638+0000", 2024-09-21T10:33:23.535 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.132399348 2024-09-21T10:33:23.535 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.535 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.535 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.797645+0000", 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3659999999999997e-06 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799536+0000", 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0018909129999999999 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799569+0000", 2024-09-21T10:33:23.536 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3500000000000001e-05 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:22.799815+0000", 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00024582199999999998 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:171427 9.d 9:b369ed47:::benchmark_data_smithi032_265250_object43625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:58.579006+0000", 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 384.912212168, 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.164263189, 2024-09-21T10:33:23.537 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 171427 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579006+0000", 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.538 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579006+0000", 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579005+0000", 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579011+0000", 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9330000000000004e-06 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.539 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579028+0000", 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.6518999999999999e-05 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579036+0000", 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6869999999999994e-06 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579113+0000", 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6953000000000007e-05 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.540 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579240+0000", 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012727699999999999 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.579369+0000", 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000129498 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.583851+0000", 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0044817190000000003 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.541 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.542 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.542 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-09-21T10:33:23.542 INFO:teuthology.orchestra.run.smithi032.stdout: "ops": [], 2024-09-21T10:33:23.542 INFO:teuthology.orchestra.run.smithi032.stdout: "num_ops": 0 2024-09-21T10:33:23.543 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-09-21T10:33:23.543 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.583859+0000", 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.4549999999999998e-06 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.743172+0000", 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15931371899999999 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.743228+0000", 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5472e-05 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.544 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.743269+0000", 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.1585999999999998e-05 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:171437 9.d 9:b73bf904:::benchmark_data_smithi032_265250_object43635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:58.592651+0000", 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 384.89856691300002, 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15252275100000001, 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.545 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 171437 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592651+0000", 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.546 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592651+0000", 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592650+0000", 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592655+0000", 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7090000000000001e-06 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.547 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592663+0000", 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.011e-06 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592668+0000", 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4240000000000003e-06 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592698+0000", 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.9977000000000001e-05 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.548 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592783+0000", 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5944000000000002e-05 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592863+0000", 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.9072000000000005e-05 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.596988+0000", 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0041250569999999997 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.549 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.597692+0000", 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00070395799999999997 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745079+0000", 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.147387925 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745124+0000", 2024-09-21T10:33:23.550 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.4453000000000003e-05 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745174+0000", 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0219999999999997e-05 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:171438 9.d 9:b6209aad:::benchmark_data_smithi032_265250_object43636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:26:58.592704+0000", 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 384.89851404000001, 2024-09-21T10:33:23.551 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15258337299999999, 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 171438 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592704+0000", 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.552 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592704+0000", 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592704+0000", 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.553 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592706+0000", 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.272e-06 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592712+0000", 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.3809999999999998e-06 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.592715+0000", 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.2770000000000001e-06 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.554 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.595442+0000", 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0027276430000000001 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.595510+0000", 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.7891000000000004e-05 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.595577+0000", 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.6830999999999998e-05 2024-09-21T10:33:23.555 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.597115+0000", 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0015379 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.597720+0000", 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00060456099999999999 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745217+0000", 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14749691500000001 2024-09-21T10:33:23.556 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745256+0000", 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.9857000000000001e-05 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:26:58.745288+0000", 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.1208999999999999e-05 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.557 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.558 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.558 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186530 9.a e404/378 9:5f76c579:::benchmark_data_smithi032_265250_object58728:head v 404'11664, mlcod=404'11661)", 2024-09-21T10:33:23.558 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.943986+0000", 2024-09-21T10:33:23.558 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 365.54723238499997, 2024-09-21T10:33:23.558 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.140619997, 2024-09-21T10:33:23.558 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.558 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.558 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.558 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.943986+0000", 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.943986+0000", 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.943985+0000", 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.559 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.943992+0000", 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4819999999999999e-06 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.944003+0000", 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1539000000000001e-05 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.944010+0000", 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.0310000000000001e-06 2024-09-21T10:33:23.560 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.944379+0000", 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000368826 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.944398+0000", 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8862e-05 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950107+0000", 2024-09-21T10:33:23.561 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.005708879 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084606+0000", 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13449923699999999 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:186535 9.d 9:b402cb59:::benchmark_data_smithi032_265250_object58733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e404)", 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.945797+0000", 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 365.54542087499999, 2024-09-21T10:33:23.562 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.142660606, 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 186535 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945797+0000", 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.563 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945797+0000", 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945796+0000", 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945802+0000", 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.3260000000000001e-06 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.564 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945814+0000", 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2048999999999999e-05 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945821+0000", 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.5749999999999997e-06 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945922+0000", 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000100874 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.565 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.946026+0000", 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00010377100000000001 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.946127+0000", 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000101897 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084660+0000", 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.138532405 2024-09-21T10:33:23.566 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084668+0000", 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8029999999999997e-06 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088340+0000", 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.003672168 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088412+0000", 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2311999999999995e-05 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.567 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088458+0000", 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5905000000000002e-05 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186534 9.7 e404/378 9:e7392b0d:::benchmark_data_smithi032_265250_object58732:head v 404'11653, mlcod=404'11650)", 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.945845+0000", 2024-09-21T10:33:23.568 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 365.54537285999999, 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13880384200000001, 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945845+0000", 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.569 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945845+0000", 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945845+0000", 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945849+0000", 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.2599999999999999e-06 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.570 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945856+0000", 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9670000000000004e-06 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945861+0000", 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7639999999999996e-06 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945907+0000", 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.5587000000000002e-05 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.571 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.945925+0000", 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.8039000000000001e-05 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084615+0000", 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13869034099999999 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084649+0000", 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4279999999999997e-05 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.572 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186536 9.c e404/378 9:3357cb79:::benchmark_data_smithi032_265250_object58734:head v 404'11671, mlcod=404'11669)", 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.947568+0000", 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 365.54365048300002, 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.140591682, 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947568+0000", 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.573 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947568+0000", 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947567+0000", 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947575+0000", 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.4880000000000007e-06 2024-09-21T10:33:23.574 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947587+0000", 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1748e-05 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947594+0000", 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.391e-06 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.575 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947655+0000", 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0457000000000001e-05 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.947675+0000", 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.9763e-05 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088122+0000", 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14044704799999999 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.576 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088160+0000", 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7774000000000001e-05 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186528 9.5 e404/390 9:a6579e44:::benchmark_data_smithi032_265250_object58726:head v 404'11718, mlcod=385'5737)", 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.950012+0000", 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 365.541206182, 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.15133627099999999, 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.577 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950012+0000", 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950012+0000", 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950011+0000", 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:23.578 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950018+0000", 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4219999999999997e-06 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950028+0000", 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6889999999999999e-06 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950033+0000", 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.8459999999999996e-06 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.579 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950149+0000", 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000115339 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.950169+0000", 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.0564000000000002e-05 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.099276+0000", 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14910671 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.580 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.101348+0000", 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002072384 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_repop(client.45848.0:186538 9.c e404/378 9:3246b83d:::benchmark_data_smithi032_265250_object58736:head v 404'11672, mlcod=404'11669)", 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.951145+0000", 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 365.54007325100002, 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.144223773, 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.581 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.951145+0000", 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.951145+0000", 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.951144+0000", 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.999999 2024-09-21T10:33:23.582 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.951151+0000", 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.8469999999999998e-06 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084575+0000", 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13342368399999999 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084583+0000", 2024-09-21T10:33:23.583 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3049999999999993e-06 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084753+0000", 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00016996399999999999 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084777+0000", 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.4063999999999999e-05 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095335+0000", 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010557715000000001 2024-09-21T10:33:23.584 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095369+0000", 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.4022000000000001e-05 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.45848.0:186541 9.e 9:72d26f2c:::benchmark_data_smithi032_265250_object58739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e404)", 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:27:17.953425+0000", 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 365.53779357899998, 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.14240792499999999, 2024-09-21T10:33:23.585 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.45848", 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/3587482455", 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 186541 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.953425+0000", 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.586 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.953425+0000", 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.953423+0000", 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999981 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:17.953431+0000", 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8909999999999995e-06 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.587 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084557+0000", 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.13112615699999999 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084573+0000", 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5968e-05 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084620+0000", 2024-09-21T10:33:23.588 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.6789000000000002e-05 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084748+0000", 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00012812000000000001 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.084879+0000", 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013129 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.088729+0000", 2024-09-21T10:33:23.589 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0038496479999999998 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.091647+0000", 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0029184129999999999 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095709+0000", 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0040617040000000002 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.590 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095793+0000", 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3704999999999999e-05 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:27:18.095833+0000", 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.0144000000000001e-05 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:45866 10.c 10:3b641306:::benchmark_data_smithi032_297285_object45865:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-09-21T10:33:23.591 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:51.556348+0000", 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 211.93487069400001, 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.18469603100000001, 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 45866 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556348+0000", 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.592 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556348+0000", 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556347+0000", 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556403+0000", 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5930000000000002e-05 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.593 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556418+0000", 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.5174000000000001e-05 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.556424+0000", 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0290000000000002e-06 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.558347+0000", 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0019228330000000001 2024-09-21T10:33:23.594 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.558514+0000", 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000167446 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.558657+0000", 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014288500000000001 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.562496+0000", 2024-09-21T10:33:23.595 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0038387500000000001 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.562557+0000", 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.0871e-05 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.740917+0000", 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.17835991900000001 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.740994+0000", 2024-09-21T10:33:23.596 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.6959000000000006e-05 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.741044+0000", 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.9864999999999998e-05 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:45869 10.c 10:39383c1a:::benchmark_data_smithi032_297285_object45868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:51.557825+0000", 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 211.93339356000001, 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.189745101, 2024-09-21T10:33:23.597 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 45869 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557825+0000", 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.598 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557825+0000", 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557823+0000", 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557878+0000", 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.4939999999999999e-05 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.599 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557893+0000", 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.4817999999999999e-05 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.557900+0000", 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9460000000000001e-06 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.560539+0000", 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.002638376 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.600 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.560670+0000", 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00013103 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.560790+0000", 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000120327 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.562501+0000", 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0017107489999999999 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.601 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.563165+0000", 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00066432600000000002 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.747434+0000", 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.184268722 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.747519+0000", 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.5310999999999997e-05 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.602 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.747570+0000", 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.0856999999999998e-05 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:45896 10.c 10:3be2d8d5:::benchmark_data_smithi032_297285_object45895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:29:51.577908+0000", 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 211.913310795, 2024-09-21T10:33:23.603 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.17023393000000001, 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 45896 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577908+0000", 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.604 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577908+0000", 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577906+0000", 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577958+0000", 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.2077000000000003e-05 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.605 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577970+0000", 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2072000000000001e-05 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.577977+0000", 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.2169999999999997e-06 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.578026+0000", 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.8439000000000002e-05 2024-09-21T10:33:23.606 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.578170+0000", 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014389700000000001 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.578318+0000", 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00014834200000000001 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.579725+0000", 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0014065900000000001 2024-09-21T10:33:23.607 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.608 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.608 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.608 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.579957+0000", 2024-09-21T10:33:23.608 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00023208699999999999 2024-09-21T10:33:23.609 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.609 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.609 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:33:23.609 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.748060+0000", 2024-09-21T10:33:23.609 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.16810291899999999 2024-09-21T10:33:23.609 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.609 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "commit_sent", 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.748108+0000", 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4.7919e-05 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:29:51.748141+0000", 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.3880000000000001e-05 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.610 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:185167 10.c 10:35154498:::benchmark_data_smithi032_297285_object67705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:04.944206+0000", 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 78.547012371999998, 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.248915373, 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 185167 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.611 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.944206+0000", 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.944206+0000", 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.944204+0000", 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999976 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.612 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.944215+0000", 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1372000000000001e-05 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.944239+0000", 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.3791999999999999e-05 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.944249+0000", 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.6649999999999998e-06 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.613 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.946530+0000", 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0022816580000000002 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.946702+0000", 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00017205500000000001 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4", 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947132+0000", 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00042980299999999998 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.614 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.961451+0000", 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.014318668 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.970523+0000", 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0090721199999999995 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.970534+0000", 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1372000000000001e-05 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.615 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.972151+0000", 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0016164000000001 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.972162+0000", 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.1728e-05 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.193038+0000", 2024-09-21T10:33:23.616 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2208752869999999 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.193121+0000", 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.3698999999999999e-05 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:185169 10.c 10:3c8219e6:::benchmark_data_smithi032_297285_object67707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:04.947394+0000", 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 78.543824372000003, 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2464000909999999, 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.617 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 185169 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947394+0000", 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.618 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947394+0000", 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947393+0000", 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999986 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947398+0000", 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.5600000000000001e-06 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.619 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947409+0000", 2024-09-21T10:33:23.620 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0895e-05 2024-09-21T10:33:23.620 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.620 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.620 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.620 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947417+0000", 2024-09-21T10:33:23.620 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 8.2979999999999992e-06 2024-09-21T10:33:23.620 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.620 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.620 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.950633+0000", 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0032156179999999999 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.950749+0000", 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00011557000000000001 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4", 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.950860+0000", 2024-09-21T10:33:23.621 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.000111062 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.961476+0000", 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.010616575 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.970565+0000", 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0090888380000000001 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.622 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.970571+0000", 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.4320000000000004e-06 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.972189+0000", 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.0016176939999999 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.972197+0000", 2024-09-21T10:33:23.623 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8620000000000003e-06 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.193400+0000", 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.2212034919999999 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.193794+0000", 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00039356999999999999 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.624 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "description": "osd_op(client.36234.0:185171 10.c 10:35f4eba8:::benchmark_data_smithi032_297285_object67709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "initiated_at": "2024-09-21T10:32:04.947428+0000", 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "age": 78.543790572999995, 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 2.2461603970000001, 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "type_data": { 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "flag_point": "reached pg", 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "client_info": { 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "client": "client.36234", 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "client_addr": "172.21.15.32:0/1953428398", 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "tid": 185171 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: "events": [ 2024-09-21T10:33:23.625 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "initiated", 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947428+0000", 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "throttled", 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947428+0000", 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "header_read", 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947427+0000", 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 4294967295.9999995 2024-09-21T10:33:23.626 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "all_read", 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947431+0000", 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 3.7790000000000002e-06 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "dispatched", 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947438+0000", 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 6.9290000000000001e-06 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "queued_for_pg", 2024-09-21T10:33:23.627 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.947443+0000", 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.1909999999999999e-06 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.956179+0000", 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0087360179999999999 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "started", 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.956767+0000", 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00058750600000000001 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.628 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for subops from 4", 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.957241+0000", 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.00047436100000000002 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "op_commit", 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.961787+0000", 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.0045457509999999998 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.972928+0000", 2024-09-21T10:33:23.629 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.011141224 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:04.972937+0000", 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 9.2790000000000004e-06 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.972223+0000", 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 0.99928577399999996 2024-09-21T10:33:23.630 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "waiting for peered", 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:05.972231+0000", 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 7.8229999999999995e-06 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "reached_pg", 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.193529+0000", 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 1.221298057 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-21T10:33:23.631 INFO:teuthology.orchestra.run.smithi045.stdout: "event": "done", 2024-09-21T10:33:23.632 INFO:teuthology.orchestra.run.smithi045.stdout: "time": "2024-09-21T10:32:07.193588+0000", 2024-09-21T10:33:23.632 INFO:teuthology.orchestra.run.smithi045.stdout: "duration": 5.9256000000000003e-05 2024-09-21T10:33:23.632 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.632 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.632 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.632 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-21T10:33:23.632 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-21T10:33:23.632 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:33:23.632 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:23.654 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:23.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:23.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:23.756 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:23.778 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:23.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:23.879 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:23.902 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:23.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:23.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:23.943 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:33:24.004 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:24.026 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:24.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:24.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:24.128 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:24.150 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:24.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:24.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:24.194 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:23 smithi032 ceph-mon[105748]: pgmap v1519: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:24.194 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:23 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3106346502' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:33:24.194 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:23 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:33:24.194 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:23 smithi032 ceph-mon[101299]: pgmap v1519: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:24.194 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:23 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3106346502' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:33:24.194 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:23 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:33:24.252 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:24.274 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:24.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:23 smithi045 ceph-mon[95976]: pgmap v1519: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:23 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3106346502' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:33:24.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:23 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:33:24.376 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:24.398 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:24.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:24.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:24.436 DEBUG:teuthology.orchestra.run.smithi032:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-21T10:33:24.461 INFO:tasks.workunit.client.0.smithi032.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-21T10:33:24.463 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T10:33:24.464 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-21T10:33:24.500 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:24.522 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:24.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:24.561 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:33:24.624 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:24.646 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:24.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:24.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:24.745 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-21T10:33:24.747 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:24.767 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:33:24.769 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:24.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:24.797 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-21T10:33:24.840 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T10:33:24.841 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T10:33:24.841 INFO:tasks.workunit.client.0.smithi032.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-21T10:33:24.871 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:24.881 INFO:tasks.workunit.client.0.smithi032.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T10:33:24.893 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:24.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:24.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:24.905 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-09-21T10:33:24.905 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-21T10:33:24.988 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-09-21T10:33:24.989 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-21T10:33:24.990 DEBUG:teuthology.orchestra.run.smithi032:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-21T10:33:24.994 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:25.051 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:25.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:25.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:25.145 INFO:tasks.workunit.client.0.smithi032.stdout:Running main() from gmock_main.cc 2024-09-21T10:33:25.145 INFO:tasks.workunit.client.0.smithi032.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-21T10:33:25.145 INFO:tasks.workunit.client.0.smithi032.stdout:[----------] Global test environment set-up. 2024-09-21T10:33:25.145 INFO:tasks.workunit.client.0.smithi032.stdout:[----------] 70 tests from TestClsRbd 2024-09-21T10:33:25.153 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:25.176 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:25.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:25.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:24 smithi032 ceph-mon[101299]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:33:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:24 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:33:25.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:24 smithi032 ceph-mon[101299]: osdmap e495: 8 total, 8 up, 7 in 2024-09-21T10:33:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:24 smithi032 ceph-mon[105748]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:33:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:24 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:33:25.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:24 smithi032 ceph-mon[105748]: osdmap e495: 8 total, 8 up, 7 in 2024-09-21T10:33:25.278 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:24 smithi045 ceph-mon[95976]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:33:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:24 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:33:25.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:24 smithi045 ceph-mon[95976]: osdmap e495: 8 total, 8 up, 7 in 2024-09-21T10:33:25.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:25.301 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:25.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:25.402 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:25.424 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:25.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:25.526 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:25.534 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:25.547 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:25.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:25.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:25.649 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:25.671 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:25.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:25.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:25.773 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:25.795 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:25.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:25.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:25.897 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:25.919 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:25.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:25.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:26.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:26.043 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:26.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:26.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:26.144 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:26.167 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:26.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:26.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:25 smithi032 ceph-mon[101299]: pgmap v1521: 93 pgs: 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:25 smithi032 ceph-mon[105748]: pgmap v1521: 93 pgs: 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:26.268 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:26.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:25 smithi045 ceph-mon[95976]: pgmap v1521: 93 pgs: 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:26.291 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:26.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:26.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:26.392 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:26.415 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:26.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:26.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:26.517 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:26.540 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:26.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:26.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:26.642 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:26.666 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:26.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:26.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:26.768 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:26.791 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:26.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:26.892 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:26.916 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:26.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:26.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:26.963 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-21T10:33:26.993 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.get_all_features (31 ms) 2024-09-21T10:33:26.993 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.copyup 2024-09-21T10:33:27.018 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:27.040 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:27.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:27.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:27.045 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:26 smithi032 ceph-mon[101299]: osdmap e496: 8 total, 8 up, 7 in 2024-09-21T10:33:27.045 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:26 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/332517685' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi032-322511-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:33:27.046 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:26 smithi032 ceph-mon[105748]: osdmap e496: 8 total, 8 up, 7 in 2024-09-21T10:33:27.046 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:26 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/332517685' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi032-322511-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:33:27.106 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:33:27.142 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:27.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:27.172 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:27.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:27.183 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.copyup (190 ms) 2024-09-21T10:33:27.183 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-21T10:33:27.196 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.sparse_copyup (13 ms) 2024-09-21T10:33:27.196 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-21T10:33:27.203 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-09-21T10:33:27.203 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-21T10:33:27.213 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2024-09-21T10:33:27.213 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-21T10:33:27.240 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.directory_methods (26 ms) 2024-09-21T10:33:27.240 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.create 2024-09-21T10:33:27.273 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:27.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:26 smithi045 ceph-mon[95976]: osdmap e496: 8 total, 8 up, 7 in 2024-09-21T10:33:27.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:26 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/332517685' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi032-322511-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:33:27.296 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:27.296 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.create (57 ms) 2024-09-21T10:33:27.297 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.get_features 2024-09-21T10:33:27.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:27.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:27.298 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-21T10:33:27.298 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-21T10:33:27.300 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-09-21T10:33:27.300 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-21T10:33:27.301 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2024-09-21T10:33:27.301 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-21T10:33:27.302 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2024-09-21T10:33:27.302 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-21T10:33:27.304 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2024-09-21T10:33:27.304 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-21T10:33:27.307 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-09-21T10:33:27.307 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.get_size 2024-09-21T10:33:27.310 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.get_size (3 ms) 2024-09-21T10:33:27.311 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.set_size 2024-09-21T10:33:27.315 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2024-09-21T10:33:27.315 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-21T10:33:27.332 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2024-09-21T10:33:27.333 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-21T10:33:27.339 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-09-21T10:33:27.339 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-21T10:33:27.382 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.parents_v1 (43 ms) 2024-09-21T10:33:27.382 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-21T10:33:27.398 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:27.413 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.parents_v2 (31 ms) 2024-09-21T10:33:27.414 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-21T10:33:27.420 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:27.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:27.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:27.436 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.snapshots (23 ms) 2024-09-21T10:33:27.437 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-21T10:33:27.440 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.snapid_race (4 ms) 2024-09-21T10:33:27.440 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-21T10:33:27.448 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2024-09-21T10:33:27.449 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-21T10:33:27.449 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2024-09-21T10:33:27.450 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-21T10:33:27.454 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.object_map_resize (4 ms) 2024-09-21T10:33:27.454 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-21T10:33:27.459 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.object_map_update (6 ms) 2024-09-21T10:33:27.459 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-21T10:33:27.459 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-21T10:33:27.460 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-21T10:33:27.463 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2024-09-21T10:33:27.463 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-21T10:33:27.467 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.object_map_snap_remove (3 ms) 2024-09-21T10:33:27.467 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.flags 2024-09-21T10:33:27.471 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-09-21T10:33:27.472 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.metadata 2024-09-21T10:33:27.496 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.metadata (25 ms) 2024-09-21T10:33:27.496 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.set_features 2024-09-21T10:33:27.503 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-09-21T10:33:27.503 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.mirror 2024-09-21T10:33:27.522 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:27.538 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.mirror (35 ms) 2024-09-21T10:33:27.539 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-21T10:33:27.548 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:27.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:27.554 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2024-09-21T10:33:27.554 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-21T10:33:27.649 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:27.675 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:27.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:27.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:27.776 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:27.800 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:27.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:27.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:27.902 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:27.929 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:27.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:27.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:28.030 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:28.053 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:28.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:28.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:28.154 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:28.177 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:28.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:28.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:27 smithi032 ceph-mon[101299]: pgmap v1523: 125 pgs: 4 creating+peering, 28 unknown, 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:28.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:27 smithi032 ceph-mon[101299]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:33:28.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/332517685' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi032-322511-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:33:28.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:27 smithi032 ceph-mon[101299]: osdmap e497: 8 total, 8 up, 7 in 2024-09-21T10:33:28.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:27 smithi032 ceph-mon[105748]: pgmap v1523: 125 pgs: 4 creating+peering, 28 unknown, 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:28.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:27 smithi032 ceph-mon[105748]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:33:28.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/332517685' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi032-322511-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:33:28.192 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:27 smithi032 ceph-mon[105748]: osdmap e497: 8 total, 8 up, 7 in 2024-09-21T10:33:28.261 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:27 smithi045 ceph-mon[95976]: pgmap v1523: 125 pgs: 4 creating+peering, 28 unknown, 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:28.261 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:27 smithi045 ceph-mon[95976]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:33:28.261 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/332517685' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi032-322511-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:33:28.261 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:27 smithi045 ceph-mon[95976]: osdmap e497: 8 total, 8 up, 7 in 2024-09-21T10:33:28.278 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:28.307 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:28.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:28.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:28.409 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:28.434 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:28.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:28.535 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:28.559 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:28.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:28.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:28.661 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:28.684 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:28.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:28.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:28.685 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:28.786 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:28.809 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:28.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:28.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:28.831 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-21T10:33:28.831 INFO:teuthology.orchestra.run.smithi045.stdout: "ops": [], 2024-09-21T10:33:28.831 INFO:teuthology.orchestra.run.smithi045.stdout: "num_ops": 0 2024-09-21T10:33:28.831 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-21T10:33:28.911 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:28.934 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:28.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:28.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:29.036 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:29.062 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:29.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:29.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:29.164 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:29.186 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:29.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:29.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:29.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:28 smithi032 ceph-mon[101299]: osdmap e498: 8 total, 8 up, 7 in 2024-09-21T10:33:29.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:28 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:29.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:28 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:29.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:28 smithi032 ceph-mon[105748]: osdmap e498: 8 total, 8 up, 7 in 2024-09-21T10:33:29.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:28 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:29.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:28 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:29.211 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:28 smithi045 ceph-mon[95976]: osdmap e498: 8 total, 8 up, 7 in 2024-09-21T10:33:29.211 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:28 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:29.211 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:28 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:29.288 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:29.311 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:29.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:29.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:29.412 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:29.437 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:29.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:29.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:29.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:29.564 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:29.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:29.586 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:33:29.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:29.689 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:29.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:29.790 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:29.805 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:33:29.813 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:29.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:29.915 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:29.920 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.mirror_image_status (2366 ms) 2024-09-21T10:33:29.920 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-21T10:33:29.938 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:29.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:29.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:29.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:33:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:33:29] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-09-21T10:33:30.002 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.mirror_image_map (81 ms) 2024-09-21T10:33:30.002 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-21T10:33:30.009 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-09-21T10:33:30.009 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-21T10:33:30.020 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2024-09-21T10:33:30.020 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-21T10:33:30.022 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-09-21T10:33:30.023 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-21T10:33:30.024 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-21T10:33:30.024 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-21T10:33:30.027 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-21T10:33:30.027 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-21T10:33:30.032 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2024-09-21T10:33:30.032 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-21T10:33:30.035 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-21T10:33:30.035 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-21T10:33:30.039 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-09-21T10:33:30.039 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-21T10:33:30.039 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:30.042 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-21T10:33:30.042 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-21T10:33:30.044 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-21T10:33:30.044 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-21T10:33:30.047 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_image_list (2 ms) 2024-09-21T10:33:30.047 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-21T10:33:30.051 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2024-09-21T10:33:30.051 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-21T10:33:30.053 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-09-21T10:33:30.053 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-21T10:33:30.056 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-21T10:33:30.056 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-21T10:33:30.059 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2024-09-21T10:33:30.059 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-21T10:33:30.061 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-21T10:33:30.061 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-21T10:33:30.062 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:30.063 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-21T10:33:30.063 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-21T10:33:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:30.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:30.065 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2024-09-21T10:33:30.065 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-21T10:33:30.068 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-09-21T10:33:30.068 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-21T10:33:30.070 INFO:tasks.workunit.client.0.smithi032.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-871-g6a1bdf85/rpm/el9/BUILD/ceph-18.2.4-871-g6a1bdf85/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-21T10:33:30.070 INFO:tasks.workunit.client.0.smithi032.stdout:Expected equality of these values: 2024-09-21T10:33:30.070 INFO:tasks.workunit.client.0.smithi032.stdout: 1U 2024-09-21T10:33:30.071 INFO:tasks.workunit.client.0.smithi032.stdout: Which is: 1 2024-09-21T10:33:30.071 INFO:tasks.workunit.client.0.smithi032.stdout: keys.size() 2024-09-21T10:33:30.071 INFO:tasks.workunit.client.0.smithi032.stdout: Which is: 3 2024-09-21T10:33:30.071 INFO:tasks.workunit.client.0.smithi032.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-09-21T10:33:30.071 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-21T10:33:30.073 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-21T10:33:30.073 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-21T10:33:30.090 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (17 ms) 2024-09-21T10:33:30.090 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-21T10:33:30.165 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:30.189 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:30.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:30.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:30.192 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:29 smithi045 ceph-mon[95976]: pgmap v1526: 125 pgs: 4 creating+peering, 20 unknown, 101 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 167 KiB/s rd, 74 KiB/s wr, 265 op/s 2024-09-21T10:33:30.192 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:29 smithi045 ceph-mon[95976]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:33:30.290 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:30.293 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (203 ms) 2024-09-21T10:33:30.293 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-21T10:33:30.296 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-09-21T10:33:30.296 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-21T10:33:30.298 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-21T10:33:30.298 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-21T10:33:30.305 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-09-21T10:33:30.305 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.op_features 2024-09-21T10:33:30.312 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2024-09-21T10:33:30.312 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-21T10:33:30.313 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:30.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:30.331 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.clone_parent (19 ms) 2024-09-21T10:33:30.331 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-21T10:33:30.339 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-21T10:33:30.339 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-21T10:33:30.352 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2024-09-21T10:33:30.352 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-21T10:33:30.358 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2024-09-21T10:33:30.358 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.migration 2024-09-21T10:33:30.367 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-09-21T10:33:30.367 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-21T10:33:30.374 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.migration_v1 (7 ms) 2024-09-21T10:33:30.375 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-21T10:33:30.415 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:30.437 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:30.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:30.452 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:29 smithi032 ceph-mon[105748]: pgmap v1526: 125 pgs: 4 creating+peering, 20 unknown, 101 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 167 KiB/s rd, 74 KiB/s wr, 265 op/s 2024-09-21T10:33:30.452 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:29 smithi032 ceph-mon[105748]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:33:30.452 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:29 smithi032 ceph-mon[101299]: pgmap v1526: 125 pgs: 4 creating+peering, 20 unknown, 101 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 167 KiB/s rd, 74 KiB/s wr, 265 op/s 2024-09-21T10:33:30.452 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:29 smithi032 ceph-mon[101299]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:33:30.539 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:30.562 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:30.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:30.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:30.663 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:30.686 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:30.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:30.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:30.788 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:30.810 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:30.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:30.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:30.912 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:33:30.929 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0: {} 2024-09-21T10:33:30.929 INFO:teuthology.orchestra.run.smithi032.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:33:30.933 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:30.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:30.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:33:30.941 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1: {} 2024-09-21T10:33:30.941 INFO:teuthology.orchestra.run.smithi032.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:33:30.957 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2: {} 2024-09-21T10:33:30.957 INFO:teuthology.orchestra.run.smithi032.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:33:30.967 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4, 1] out_osds: [0] dead_osds: [] live_osds: [1, 2, 3, 4, 6, 7, 5, 0] 2024-09-21T10:33:30.967 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:33:30.967 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-09-21T10:33:30.967 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd in 0 2024-09-21T10:33:30.975 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3: {} 2024-09-21T10:33:30.975 INFO:teuthology.orchestra.run.smithi032.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:33:30.996 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4: {} 2024-09-21T10:33:30.996 INFO:teuthology.orchestra.run.smithi032.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:33:31.013 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5: {} 2024-09-21T10:33:31.013 INFO:teuthology.orchestra.run.smithi032.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:33:31.029 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6: {} 2024-09-21T10:33:31.029 INFO:teuthology.orchestra.run.smithi032.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:33:31.034 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:31.048 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7: {} 2024-09-21T10:33:31.049 INFO:teuthology.orchestra.run.smithi032.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:33:31.058 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:31.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:31.159 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:33:31.182 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:31.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:31.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:33:31.220 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:30 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3518138896' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:33:31.220 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:30 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3518138896' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:33:31.284 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:33:31.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:30 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3518138896' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:33:31.308 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:31.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:31.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:33:31.410 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:31.431 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:31.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:31.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:31.533 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:31.556 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:31.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:31.562 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:33:31.562 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:33:31.658 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:31.681 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:31.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:31.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:31.782 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:31.804 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:31.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:31.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:31.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:31.928 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:31.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:31.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:31.990 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1616 ms) 2024-09-21T10:33:31.990 INFO:tasks.workunit.client.0.smithi032.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-21T10:33:32.018 INFO:tasks.workunit.client.0.smithi032.stdout:[ OK ] TestClsRbd.sparsify (28 ms) 2024-09-21T10:33:32.023 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:32.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:32.054 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:32.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:32.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:32.147 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[105748]: pgmap v1527: 125 pgs: 4 creating+peering, 121 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1.5 MiB/s rd, 762 KiB/s wr, 297 op/s 2024-09-21T10:33:32.147 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[105748]: osdmap e499: 8 total, 8 up, 7 in 2024-09-21T10:33:32.147 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:33:32.147 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:33:32.147 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:32.147 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[101299]: pgmap v1527: 125 pgs: 4 creating+peering, 121 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1.5 MiB/s rd, 762 KiB/s wr, 297 op/s 2024-09-21T10:33:32.147 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[101299]: osdmap e499: 8 total, 8 up, 7 in 2024-09-21T10:33:32.147 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:33:32.147 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:33:32.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:31 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:32.156 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:32.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:32.178 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:32.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:32.247 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:31 smithi045 ceph-mon[95976]: pgmap v1527: 125 pgs: 4 creating+peering, 121 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1.5 MiB/s rd, 762 KiB/s wr, 297 op/s 2024-09-21T10:33:32.247 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:31 smithi045 ceph-mon[95976]: osdmap e499: 8 total, 8 up, 7 in 2024-09-21T10:33:32.247 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:33:32.247 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:33:32.247 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:31 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:32.268 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:33:32.280 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:33:32.332 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:32.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:32.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:33:32.434 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:32.457 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:32.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:32.473 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:33:32.559 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:32.581 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:32.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:32.682 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:32.704 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:32.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:32.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:32.806 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:33:32.829 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:32.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:32.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:33:32.930 DEBUG:teuthology.orchestra.run.smithi032:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:33:32.953 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: docker: command not found 2024-09-21T10:33:32.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:32.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi032 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:33:32.989 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:33:32.996 INFO:tasks.workunit.client.0.smithi032.stdout:[----------] 70 tests from TestClsRbd (5056 ms total) 2024-09-21T10:33:32.996 INFO:tasks.workunit.client.0.smithi032.stdout: 2024-09-21T10:33:32.996 INFO:tasks.workunit.client.0.smithi032.stdout:[----------] Global test environment tear-down 2024-09-21T10:33:32.996 INFO:tasks.workunit.client.0.smithi032.stdout:[==========] 70 tests from 1 test suite ran. (7850 ms total) 2024-09-21T10:33:32.996 INFO:tasks.workunit.client.0.smithi032.stdout:[ PASSED ] 69 tests. 2024-09-21T10:33:32.996 INFO:tasks.workunit.client.0.smithi032.stdout:[ FAILED ] 1 test, listed below: 2024-09-21T10:33:32.996 INFO:tasks.workunit.client.0.smithi032.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-21T10:33:32.996 INFO:tasks.workunit.client.0.smithi032.stdout: 2024-09-21T10:33:32.996 INFO:tasks.workunit.client.0.smithi032.stdout: 1 FAILED TEST 2024-09-21T10:33:33.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:33.031 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-09-21T10:33:33.032 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-21T10:33:33.054 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:33:33.077 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:33.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:33.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:33:33.178 DEBUG:teuthology.orchestra.run.smithi045:> sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:33:33.201 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: docker: command not found 2024-09-21T10:33:33.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:33:33.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi045 with status 1: 'sudo docker kill -s 1 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:33:33.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:32 smithi032 ceph-mon[101299]: osdmap e500: 8 total, 8 up, 7 in 2024-09-21T10:33:33.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:32 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2268148827' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:33.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:32 smithi032 ceph-mon[105748]: osdmap e500: 8 total, 8 up, 7 in 2024-09-21T10:33:33.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:32 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2268148827' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:33.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:32 smithi045 ceph-mon[95976]: osdmap e500: 8 total, 8 up, 7 in 2024-09-21T10:33:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:32 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2268148827' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:33.289 INFO:tasks.thrashosds:joining thrashosds 2024-09-21T10:33:34.100 INFO:teuthology.orchestra.run.smithi045.stdout:ERROR: (22) Invalid argument 2024-09-21T10:33:34.100 INFO:teuthology.orchestra.run.smithi045.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:33:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:33 smithi045 ceph-mon[95976]: pgmap v1530: 125 pgs: 125 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 3.0 MiB/s rd, 1.4 MiB/s wr, 2.38k op/s 2024-09-21T10:33:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:33 smithi045 ceph-mon[95976]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:33:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:33 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:33:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:33 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2268148827' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:33:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:33 smithi045 ceph-mon[95976]: osdmap e501: 8 total, 8 up, 7 in 2024-09-21T10:33:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:33 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:34.392 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[101299]: pgmap v1530: 125 pgs: 125 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 3.0 MiB/s rd, 1.4 MiB/s wr, 2.38k op/s 2024-09-21T10:33:34.392 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[101299]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:33:34.392 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:33:34.392 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2268148827' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:33:34.392 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[101299]: osdmap e501: 8 total, 8 up, 7 in 2024-09-21T10:33:34.392 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:34.392 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:34.393 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[105748]: pgmap v1530: 125 pgs: 125 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 3.0 MiB/s rd, 1.4 MiB/s wr, 2.38k op/s 2024-09-21T10:33:34.393 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[105748]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:33:34.393 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:33:34.393 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2268148827' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:33:34.393 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[105748]: osdmap e501: 8 total, 8 up, 7 in 2024-09-21T10:33:34.393 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:33:34.393 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:33 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:34.793 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:33:35.008 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:35.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:35 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:33:35.114+0000 7f532c355640 -1 mon.a@0(leader).osd e501 definitely_dead 0 2024-09-21T10:33:35.998 INFO:teuthology.orchestra.run.smithi032.stderr:marked in osd.0. 2024-09-21T10:33:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:35 smithi045 ceph-mon[95976]: pgmap v1532: 93 pgs: 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:35 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3372345544' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-21T10:33:36.421 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:33:36.451 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:35 smithi032 ceph-mon[105748]: pgmap v1532: 93 pgs: 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:36.451 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:35 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3372345544' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-21T10:33:36.451 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:35 smithi032 ceph-mon[101299]: pgmap v1532: 93 pgs: 93 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:36.451 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:35 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3372345544' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-21T10:33:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:37 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3372345544' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-21T10:33:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:37 smithi032 ceph-mon[101299]: osdmap e502: 8 total, 8 up, 8 in 2024-09-21T10:33:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:37 smithi032 ceph-mon[101299]: osdmap e503: 8 total, 8 up, 8 in 2024-09-21T10:33:37.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:37 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3372345544' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-21T10:33:37.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:37 smithi032 ceph-mon[105748]: osdmap e502: 8 total, 8 up, 8 in 2024-09-21T10:33:37.208 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:37 smithi032 ceph-mon[105748]: osdmap e503: 8 total, 8 up, 8 in 2024-09-21T10:33:37.263 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-09-21T10:33:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:37 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3372345544' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-21T10:33:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:37 smithi045 ceph-mon[95976]: osdmap e502: 8 total, 8 up, 8 in 2024-09-21T10:33:37.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:37 smithi045 ceph-mon[95976]: osdmap e503: 8 total, 8 up, 8 in 2024-09-21T10:33:37.738 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:33:37.738 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:33:38.003 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:38 smithi045 ceph-mon[95976]: pgmap v1534: 93 pgs: 1 peering, 92 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:38.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:38 smithi045 ceph-mon[95976]: osdmap e504: 8 total, 8 up, 8 in 2024-09-21T10:33:38.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:38 smithi032 ceph-mon[101299]: pgmap v1534: 93 pgs: 1 peering, 92 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:38.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:38 smithi032 ceph-mon[101299]: osdmap e504: 8 total, 8 up, 8 in 2024-09-21T10:33:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:38 smithi032 ceph-mon[105748]: pgmap v1534: 93 pgs: 1 peering, 92 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:33:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:38 smithi032 ceph-mon[105748]: osdmap e504: 8 total, 8 up, 8 in 2024-09-21T10:33:38.513 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:33:39.215 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:39.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:33:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:33:39] "GET /metrics HTTP/1.1" 200 34147 "" "Prometheus/2.51.0" 2024-09-21T10:33:40.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:39 smithi032 ceph-mon[101299]: pgmap v1537: 93 pgs: 2 remapped+peering, 2 unknown, 12 peering, 77 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:40.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:39 smithi032 ceph-mon[101299]: osdmap e505: 8 total, 8 up, 8 in 2024-09-21T10:33:40.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:39 smithi032 ceph-mon[101299]: Health check failed: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T10:33:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:39 smithi032 ceph-mon[105748]: pgmap v1537: 93 pgs: 2 remapped+peering, 2 unknown, 12 peering, 77 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:39 smithi032 ceph-mon[105748]: osdmap e505: 8 total, 8 up, 8 in 2024-09-21T10:33:40.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:39 smithi032 ceph-mon[105748]: Health check failed: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T10:33:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:39 smithi045 ceph-mon[95976]: pgmap v1537: 93 pgs: 2 remapped+peering, 2 unknown, 12 peering, 77 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:39 smithi045 ceph-mon[95976]: osdmap e505: 8 total, 8 up, 8 in 2024-09-21T10:33:40.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:39 smithi045 ceph-mon[95976]: Health check failed: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T10:33:41.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:40 smithi045 ceph-mon[95976]: osdmap e506: 8 total, 8 up, 8 in 2024-09-21T10:33:41.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:40 smithi032 ceph-mon[101299]: osdmap e506: 8 total, 8 up, 8 in 2024-09-21T10:33:41.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:40 smithi032 ceph-mon[105748]: osdmap e506: 8 total, 8 up, 8 in 2024-09-21T10:33:42.118 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:41 smithi032 ceph-mon[105748]: pgmap v1540: 93 pgs: 2 remapped+peering, 2 unknown, 17 peering, 72 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.1 MiB/s, 9 objects/s recovering 2024-09-21T10:33:42.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:41 smithi032 ceph-mon[101299]: pgmap v1540: 93 pgs: 2 remapped+peering, 2 unknown, 17 peering, 72 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.1 MiB/s, 9 objects/s recovering 2024-09-21T10:33:42.264 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-09-21T10:33:42.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:41 smithi045 ceph-mon[95976]: pgmap v1540: 93 pgs: 2 remapped+peering, 2 unknown, 17 peering, 72 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.1 MiB/s, 9 objects/s recovering 2024-09-21T10:33:42.341 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:33:42.341 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:33:42.992 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:33:43.004 INFO:teuthology.orchestra.run.smithi032.stderr:noscrub is unset 2024-09-21T10:33:43.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:43 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2105055990' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:33:43.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:43 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2105055990' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:33:43.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:43 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2105055990' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:33:43.667 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:33:43.712 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:43.748 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:44 smithi045 ceph-mon[95976]: pgmap v1541: 93 pgs: 5 peering, 88 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 761 KiB/s, 8 objects/s recovering 2024-09-21T10:33:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:44 smithi045 ceph-mon[95976]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-21T10:33:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:44 smithi045 ceph-mon[95976]: Cluster is now healthy 2024-09-21T10:33:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:44 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2105055990' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:33:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:44 smithi045 ceph-mon[95976]: osdmap e507: 8 total, 8 up, 8 in 2024-09-21T10:33:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[101299]: pgmap v1541: 93 pgs: 5 peering, 88 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 761 KiB/s, 8 objects/s recovering 2024-09-21T10:33:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[101299]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-21T10:33:44.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[101299]: Cluster is now healthy 2024-09-21T10:33:44.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2105055990' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:33:44.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[101299]: osdmap e507: 8 total, 8 up, 8 in 2024-09-21T10:33:44.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[105748]: pgmap v1541: 93 pgs: 5 peering, 88 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 761 KiB/s, 8 objects/s recovering 2024-09-21T10:33:44.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[105748]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-21T10:33:44.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[105748]: Cluster is now healthy 2024-09-21T10:33:44.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2105055990' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:33:44.459 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:44 smithi032 ceph-mon[105748]: osdmap e507: 8 total, 8 up, 8 in 2024-09-21T10:33:45.622 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:46.103 INFO:teuthology.orchestra.run.smithi032.stdout:mon.a: {} 2024-09-21T10:33:46.103 INFO:teuthology.orchestra.run.smithi032.stdout:mon.b: {} 2024-09-21T10:33:46.103 INFO:teuthology.orchestra.run.smithi032.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:33:46.103 INFO:teuthology.orchestra.run.smithi032.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:33:46.109 INFO:teuthology.orchestra.run.smithi032.stdout:mon.c: {} 2024-09-21T10:33:46.109 INFO:teuthology.orchestra.run.smithi032.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:33:46.254 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:46 smithi032 ceph-mon[105748]: pgmap v1543: 93 pgs: 1 peering, 92 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 736 KiB/s, 8 objects/s recovering 2024-09-21T10:33:46.255 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:46 smithi032 ceph-mon[101299]: pgmap v1543: 93 pgs: 1 peering, 92 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 736 KiB/s, 8 objects/s recovering 2024-09-21T10:33:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:46 smithi045 ceph-mon[95976]: pgmap v1543: 93 pgs: 1 peering, 92 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 736 KiB/s, 8 objects/s recovering 2024-09-21T10:33:47.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:47 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2977370731' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:33:47.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:47 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2977370731' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:33:47.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:47 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2977370731' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:33:47.958 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:33:48.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:48 smithi032 ceph-mon[101299]: pgmap v1544: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 566 KiB/s, 6 objects/s recovering 2024-09-21T10:33:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:48 smithi032 ceph-mon[105748]: pgmap v1544: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 566 KiB/s, 6 objects/s recovering 2024-09-21T10:33:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:48 smithi045 ceph-mon[95976]: pgmap v1544: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 566 KiB/s, 6 objects/s recovering 2024-09-21T10:33:49.114 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:49 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:49.114 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:49 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:49 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:33:49.313 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:49.518 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: (22) Invalid argument 2024-09-21T10:33:49.518 INFO:teuthology.orchestra.run.smithi032.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:33:49.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:33:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:33:49] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.51.0" 2024-09-21T10:33:50.021 INFO:teuthology.orchestra.run.smithi032.stderr:nodeep-scrub is unset 2024-09-21T10:33:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:50 smithi045 ceph-mon[95976]: pgmap v1545: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 66 B/s, 1 objects/s recovering 2024-09-21T10:33:50.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:50 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2592093016' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:50.402 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:50 smithi032 ceph-mon[101299]: pgmap v1545: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 66 B/s, 1 objects/s recovering 2024-09-21T10:33:50.402 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:50 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2592093016' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:50.402 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:50 smithi032 ceph-mon[105748]: pgmap v1545: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 66 B/s, 1 objects/s recovering 2024-09-21T10:33:50.402 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:50 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2592093016' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:33:51.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:51 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2592093016' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:33:51.148 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:51 smithi032 ceph-mon[101299]: osdmap e508: 8 total, 8 up, 8 in 2024-09-21T10:33:51.148 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:51 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2592093016' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:33:51.148 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:51 smithi032 ceph-mon[105748]: osdmap e508: 8 total, 8 up, 8 in 2024-09-21T10:33:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:51 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2592093016' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:33:51.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:51 smithi045 ceph-mon[95976]: osdmap e508: 8 total, 8 up, 8 in 2024-09-21T10:33:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:52 smithi045 ceph-mon[95976]: pgmap v1547: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:52.312 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:52 smithi032 ceph-mon[105748]: pgmap v1547: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:52.312 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:52 smithi032 ceph-mon[101299]: pgmap v1547: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:53.040 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:33:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:53 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4026686743' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:33:53.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:53 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:33:53.368 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:53 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4026686743' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:33:53.368 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:53 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:33:53.368 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:53 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4026686743' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:33:53.368 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:53 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:33:53.795 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 1 1 2024-09-21T10:33:54.009 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:54.397 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:54 smithi032 ceph-mon[105748]: pgmap v1548: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:54.397 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:54 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:33:54.397 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:54 smithi032 ceph-mon[105748]: osdmap e509: 8 total, 8 up, 8 in 2024-09-21T10:33:54.397 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:54 smithi032 ceph-mon[101299]: pgmap v1548: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:54.397 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:54 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:33:54.397 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:54 smithi032 ceph-mon[101299]: osdmap e509: 8 total, 8 up, 8 in 2024-09-21T10:33:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:54 smithi045 ceph-mon[95976]: pgmap v1548: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:54 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:33:54.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:54 smithi045 ceph-mon[95976]: osdmap e509: 8 total, 8 up, 8 in 2024-09-21T10:33:56.041 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-09-21T10:33:56.342 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:56 smithi032 ceph-mon[105748]: pgmap v1550: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:56.342 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:56 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1922443199' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:33:56.342 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:56 smithi032 ceph-mon[101299]: pgmap v1550: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:56.342 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:56 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1922443199' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:33:56.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:56 smithi045 ceph-mon[95976]: pgmap v1550: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:56.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:56 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1922443199' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:33:56.743 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:33:56.953 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:33:57.346 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:57 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1922443199' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:33:57.346 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:57 smithi032 ceph-mon[101299]: osdmap e510: 8 total, 8 up, 8 in 2024-09-21T10:33:57.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:57 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1922443199' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:33:57.346 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:57 smithi032 ceph-mon[105748]: osdmap e510: 8 total, 8 up, 8 in 2024-09-21T10:33:57.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:57 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1922443199' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:33:57.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:57 smithi045 ceph-mon[95976]: osdmap e510: 8 total, 8 up, 8 in 2024-09-21T10:33:58.150 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:58 smithi032 ceph-mon[101299]: pgmap v1552: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:58.151 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:58 smithi032 ceph-mon[105748]: pgmap v1552: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:58.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:58 smithi045 ceph-mon[95976]: pgmap v1552: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:33:59.057 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:33:59.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:33:59 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3020419391' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:33:59.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:33:59 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3020419391' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:33:59.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:33:59 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3020419391' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:33:59.719 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:33:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:33:59] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.51.0" 2024-09-21T10:33:59.741 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 2 1 2024-09-21T10:33:59.948 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:00.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:00 smithi032 ceph-mon[105748]: pgmap v1553: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:00.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:00 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3020419391' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:34:00.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:00 smithi032 ceph-mon[105748]: osdmap e511: 8 total, 8 up, 8 in 2024-09-21T10:34:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:00 smithi032 ceph-mon[101299]: pgmap v1553: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:00 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3020419391' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:34:00.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:00 smithi032 ceph-mon[101299]: osdmap e511: 8 total, 8 up, 8 in 2024-09-21T10:34:00.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:00 smithi045 ceph-mon[95976]: pgmap v1553: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:00.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:00 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3020419391' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:34:00.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:00 smithi045 ceph-mon[95976]: osdmap e511: 8 total, 8 up, 8 in 2024-09-21T10:34:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:02 smithi045 ceph-mon[95976]: pgmap v1555: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:02.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:02 smithi032 ceph-mon[101299]: pgmap v1555: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:02.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:02 smithi032 ceph-mon[105748]: pgmap v1555: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:03.067 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-21T10:34:03.407 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:03 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/464303033' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:34:03.407 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:03 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/464303033' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:34:03.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:03 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/464303033' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:34:03.759 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:34:03.970 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:04.345 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:04 smithi032 ceph-mon[105748]: pgmap v1556: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:04.345 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:04 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/464303033' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:34:04.345 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:04 smithi032 ceph-mon[105748]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T10:34:04.345 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:04 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:04.345 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:04 smithi032 ceph-mon[101299]: pgmap v1556: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:04.345 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:04 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/464303033' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:34:04.345 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:04 smithi032 ceph-mon[101299]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T10:34:04.346 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:04 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:04 smithi045 ceph-mon[95976]: pgmap v1556: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:04 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/464303033' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:34:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:04 smithi045 ceph-mon[95976]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T10:34:04.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:04 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:06.074 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:34:06.365 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:06 smithi032 ceph-mon[105748]: pgmap v1558: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:06.365 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:06 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/601632327' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:06.365 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:06 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:06.366 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:06 smithi032 ceph-mon[101299]: pgmap v1558: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:06.366 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:06 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/601632327' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:06.366 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:06 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:06.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:06 smithi045 ceph-mon[95976]: pgmap v1558: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:06.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:06 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/601632327' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:06.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:06 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:06.783 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 3 1 2024-09-21T10:34:06.996 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:07.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:07 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:34:07.344 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:07 smithi032 ceph-mon[105748]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T10:34:07.344 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:07 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:34:07.344 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:07 smithi032 ceph-mon[101299]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T10:34:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:07 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:34:07.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:07 smithi045 ceph-mon[95976]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T10:34:08.410 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:08 smithi032 ceph-mon[101299]: pgmap v1560: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:08.410 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:08 smithi032 ceph-mon[105748]: pgmap v1560: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:08.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:08 smithi045 ceph-mon[95976]: pgmap v1560: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:09.090 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-09-21T10:34:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:09 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2385971889' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:09.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:09 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:09 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2385971889' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:09.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:09 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:09 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2385971889' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:09.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:09 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:34:09.748 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:34:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:34:09] "GET /metrics HTTP/1.1" 200 34146 "" "Prometheus/2.51.0" 2024-09-21T10:34:09.777 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:34:09.992 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:10.418 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:10 smithi032 ceph-mon[105748]: pgmap v1561: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:10.418 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:10 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:34:10.418 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:10 smithi032 ceph-mon[105748]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T10:34:10.418 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:10 smithi032 ceph-mon[101299]: pgmap v1561: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:10.419 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:10 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:34:10.419 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:10 smithi032 ceph-mon[101299]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T10:34:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:10 smithi045 ceph-mon[95976]: pgmap v1561: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:10 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:34:10.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:10 smithi045 ceph-mon[95976]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T10:34:12.091 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:34:12.405 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:12 smithi032 ceph-mon[101299]: pgmap v1563: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:12.405 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:12 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3700475563' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:34:12.406 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:12 smithi032 ceph-mon[105748]: pgmap v1563: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:12.406 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:12 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3700475563' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:34:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:12 smithi045 ceph-mon[95976]: pgmap v1563: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:12.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:12 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3700475563' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:34:12.809 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 4 1 2024-09-21T10:34:13.023 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:13.417 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3700475563' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:34:13.417 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:13 smithi032 ceph-mon[105748]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T10:34:13.418 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3700475563' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:34:13.418 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:13 smithi032 ceph-mon[101299]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T10:34:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3700475563' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:34:13.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:13 smithi045 ceph-mon[95976]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T10:34:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:14 smithi032 ceph-mon[101299]: pgmap v1565: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:14 smithi032 ceph-mon[105748]: pgmap v1565: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:14.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:14 smithi045 ceph-mon[95976]: pgmap v1565: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:15.111 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-21T10:34:15.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:15 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3594867182' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:34:15.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:15 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:34:15.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:15 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3594867182' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:34:15.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:15 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:34:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:15 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3594867182' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:34:15.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:15 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:34:15.899 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:34:16.115 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:16.206 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:16 smithi032 ceph-mon[101299]: pgmap v1566: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:16 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:34:16.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:16 smithi032 ceph-mon[101299]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T10:34:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:16 smithi032 ceph-mon[105748]: pgmap v1566: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:16 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:34:16.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:16 smithi032 ceph-mon[105748]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T10:34:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:16 smithi045 ceph-mon[95976]: pgmap v1566: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:16 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:34:16.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:16 smithi045 ceph-mon[95976]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T10:34:18.112 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:34:18.428 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:18 smithi032 ceph-mon[105748]: pgmap v1568: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:18.429 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:18 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2832453165' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:34:18.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:18 smithi032 ceph-mon[101299]: pgmap v1568: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:18.429 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:18 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2832453165' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:34:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:18 smithi045 ceph-mon[95976]: pgmap v1568: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:18.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:18 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2832453165' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:34:18.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 6 1 2024-09-21T10:34:19.078 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:19.449 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2832453165' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:34:19.449 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:19 smithi032 ceph-mon[101299]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T10:34:19.450 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:19 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:19.450 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2832453165' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:34:19.450 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:19 smithi032 ceph-mon[105748]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T10:34:19.450 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:19 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2832453165' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:34:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:19 smithi045 ceph-mon[95976]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T10:34:19.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:19 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:19.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:34:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:34:19] "GET /metrics HTTP/1.1" 200 34145 "" "Prometheus/2.51.0" 2024-09-21T10:34:20.436 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:20 smithi045 ceph-mon[95976]: pgmap v1570: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:20.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:20 smithi032 ceph-mon[101299]: pgmap v1570: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:20.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:20 smithi032 ceph-mon[105748]: pgmap v1570: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:21.131 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-21T10:34:21.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:21 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0]} : dispatch 2024-09-21T10:34:21.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:21 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1209690645' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:34:21.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:21 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:34:21.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:21 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0]} : dispatch 2024-09-21T10:34:21.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:21 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1209690645' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:34:21.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:21 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:34:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:21 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0]} : dispatch 2024-09-21T10:34:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:21 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1209690645' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:34:21.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:21 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:34:21.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 7 1 2024-09-21T10:34:22.080 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:22.129 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:22 smithi032 ceph-mon[101299]: pgmap v1571: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:22.386 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:22 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0]}]': finished 2024-09-21T10:34:22.386 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:22 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:34:22.386 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:22 smithi032 ceph-mon[101299]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T10:34:22.386 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:22 smithi032 ceph-mon[105748]: pgmap v1571: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:22.386 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:22 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0]}]': finished 2024-09-21T10:34:22.386 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:22 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:34:22.386 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:22 smithi032 ceph-mon[105748]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T10:34:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:22 smithi045 ceph-mon[95976]: pgmap v1571: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:22 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0]}]': finished 2024-09-21T10:34:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:22 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:34:22.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:22 smithi045 ceph-mon[95976]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T10:34:23.446 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:23 smithi032 ceph-mon[105748]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T10:34:23.446 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:23 smithi032 ceph-mon[101299]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T10:34:23.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:23 smithi045 ceph-mon[95976]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T10:34:24.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:24 smithi032 ceph-mon[101299]: pgmap v1574: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:24.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:24 smithi032 ceph-mon[105748]: pgmap v1574: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:24.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:24 smithi045 ceph-mon[95976]: pgmap v1574: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:25.151 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.7 to 1 (10000) 2024-09-21T10:34:25.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:25 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/73143647' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:25.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:25 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:25 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/73143647' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:25.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:25 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:25.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:25 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/73143647' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:25.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:25 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:25.898 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 7 1 2024-09-21T10:34:26.113 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:26.154 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:26 smithi032 ceph-mon[105748]: pgmap v1575: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:26.154 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:26 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:34:26.154 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:26 smithi032 ceph-mon[105748]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T10:34:26.155 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:26 smithi032 ceph-mon[101299]: pgmap v1575: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:26.155 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:26 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:34:26.155 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:26 smithi032 ceph-mon[101299]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T10:34:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:26 smithi045 ceph-mon[95976]: pgmap v1575: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:26 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:34:26.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:26 smithi045 ceph-mon[95976]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T10:34:27.228 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:27 smithi032 ceph-mon[101299]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T10:34:27.228 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:27 smithi032 ceph-mon[101299]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T10:34:27.228 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:27 smithi032 ceph-mon[105748]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T10:34:27.228 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:27 smithi032 ceph-mon[105748]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T10:34:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:27 smithi045 ceph-mon[95976]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T10:34:27.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:27 smithi045 ceph-mon[95976]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T10:34:28.169 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-09-21T10:34:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:28 smithi032 ceph-mon[105748]: pgmap v1578: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:28 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2451331196' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:28.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:28 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:28 smithi032 ceph-mon[101299]: pgmap v1578: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:28 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2451331196' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:28.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:28 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:28 smithi045 ceph-mon[95976]: pgmap v1578: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:28 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2451331196' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:28.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:28 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:34:28.869 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:34:29.080 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:29.171 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:29 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:34:29.456 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:29 smithi032 ceph-mon[101299]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T10:34:29.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:29 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:34:29.456 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:29 smithi032 ceph-mon[105748]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T10:34:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:29 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:34:29.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:29 smithi045 ceph-mon[95976]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T10:34:29.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:34:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:34:29] "GET /metrics HTTP/1.1" 200 34145 "" "Prometheus/2.51.0" 2024-09-21T10:34:30.440 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:30 smithi032 ceph-mon[101299]: pgmap v1581: 93 pgs: 2 remapped+peering, 91 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:30.440 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:30 smithi032 ceph-mon[101299]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T10:34:30.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:30 smithi032 ceph-mon[105748]: pgmap v1581: 93 pgs: 2 remapped+peering, 91 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:30.441 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:30 smithi032 ceph-mon[105748]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T10:34:30.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:30 smithi045 ceph-mon[95976]: pgmap v1581: 93 pgs: 2 remapped+peering, 91 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:30.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:30 smithi045 ceph-mon[95976]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T10:34:31.181 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:34:31.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:31 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2427796794' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:31.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:31 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:31.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:31 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2427796794' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:31.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:31 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:31.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:31 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2427796794' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:31.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:31 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:31.985 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 5 1 2024-09-21T10:34:32.324 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:32 smithi032 ceph-mon[101299]: pgmap v1583: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 8.2 KiB/s rd, 24 KiB/s wr, 6 op/s; 28 KiB/s, 2 objects/s recovering 2024-09-21T10:34:32.324 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:32 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:34:32.324 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:32 smithi032 ceph-mon[101299]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T10:34:32.324 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:32 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:34:32.325 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:32 smithi032 ceph-mon[105748]: pgmap v1583: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 8.2 KiB/s rd, 24 KiB/s wr, 6 op/s; 28 KiB/s, 2 objects/s recovering 2024-09-21T10:34:32.325 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:32 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:34:32.325 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:32 smithi032 ceph-mon[105748]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T10:34:32.325 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:32 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:34:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:32 smithi045 ceph-mon[95976]: pgmap v1583: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 8.2 KiB/s rd, 24 KiB/s wr, 6 op/s; 28 KiB/s, 2 objects/s recovering 2024-09-21T10:34:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:32 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:34:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:32 smithi045 ceph-mon[95976]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T10:34:32.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:32 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:34:32.926 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:34.297 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:34 smithi032 ceph-mon[105748]: pgmap v1585: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 6.7 KiB/s rd, 19 KiB/s wr, 4 op/s; 23 KiB/s, 1 objects/s recovering 2024-09-21T10:34:34.297 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:34 smithi032 ceph-mon[105748]: 1.d scrub starts 2024-09-21T10:34:34.297 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:34 smithi032 ceph-mon[105748]: 1.d scrub ok 2024-09-21T10:34:34.297 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:34.297 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:34 smithi032 ceph-mon[101299]: pgmap v1585: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 6.7 KiB/s rd, 19 KiB/s wr, 4 op/s; 23 KiB/s, 1 objects/s recovering 2024-09-21T10:34:34.297 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:34 smithi032 ceph-mon[101299]: 1.d scrub starts 2024-09-21T10:34:34.298 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:34 smithi032 ceph-mon[101299]: 1.d scrub ok 2024-09-21T10:34:34.298 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:34.509 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:34 smithi045 ceph-mon[95976]: pgmap v1585: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 6.7 KiB/s rd, 19 KiB/s wr, 4 op/s; 23 KiB/s, 1 objects/s recovering 2024-09-21T10:34:34.509 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:34 smithi045 ceph-mon[95976]: 1.d scrub starts 2024-09-21T10:34:34.509 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:34 smithi045 ceph-mon[95976]: 1.d scrub ok 2024-09-21T10:34:34.509 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:35.189 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-09-21T10:34:35.219 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:35 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4187948988' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:35.219 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:35 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:35.220 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:35 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4187948988' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:35.220 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:35 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:35 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4187948988' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:35.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:35 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:34:36.010 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd reweight 0 1 2024-09-21T10:34:36.330 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[105748]: pgmap v1586: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2024-09-21T10:34:36.330 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:34:36.330 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[105748]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T10:34:36.330 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:36.330 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:36.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[101299]: pgmap v1586: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2024-09-21T10:34:36.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:34:36.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[101299]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T10:34:36.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:36.331 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:36 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:36 smithi045 ceph-mon[95976]: pgmap v1586: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2024-09-21T10:34:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:36 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:34:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:36 smithi045 ceph-mon[95976]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T10:34:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:36 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:36.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:36 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:37.348 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:38 smithi032 ceph-mon[105748]: pgmap v1588: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2024-09-21T10:34:38.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:38 smithi032 ceph-mon[101299]: pgmap v1588: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2024-09-21T10:34:38.535 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:38 smithi045 ceph-mon[95976]: pgmap v1588: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2024-09-21T10:34:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:39 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:39.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:39 smithi045 ceph-mon[95976]: pgmap v1589: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:39.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:34:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:34:39] "GET /metrics HTTP/1.1" 200 34153 "" "Prometheus/2.51.0" 2024-09-21T10:34:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:39 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:39.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:39 smithi032 ceph-mon[105748]: pgmap v1589: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:39 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:39.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:39 smithi032 ceph-mon[101299]: pgmap v1589: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:40.477 INFO:teuthology.orchestra.run.smithi032.stderr:reweighted osd.0 to 1 (10000) 2024-09-21T10:34:40.619 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:40 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/960740349' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:34:40.620 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:40.620 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:40 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:40.620 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:40 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/960740349' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:34:40.620 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:40.620 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:40 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:40 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/960740349' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:34:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:40 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:41.206 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 0 1 2024-09-21T10:34:41.666 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:41 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/960740349' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:34:41.666 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:41 smithi032 ceph-mon[101299]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T10:34:41.667 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:41 smithi032 ceph-mon[101299]: pgmap v1591: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:41.667 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:41 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/960740349' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:34:41.667 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:41 smithi032 ceph-mon[105748]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T10:34:41.667 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:41 smithi032 ceph-mon[105748]: pgmap v1591: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:41 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/960740349' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:34:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:41 smithi045 ceph-mon[95976]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T10:34:41.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:41 smithi045 ceph-mon[95976]: pgmap v1591: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:42.159 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:43.891 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:43 smithi032 ceph-mon[105748]: pgmap v1592: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:43.891 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:43 smithi032 ceph-mon[101299]: pgmap v1592: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:44.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:43 smithi045 ceph-mon[95976]: pgmap v1592: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:44.861 INFO:teuthology.orchestra.run.smithi032.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-21T10:34:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:44 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3394381607' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:34:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:44 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:34:45.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:44 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:44 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3394381607' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:34:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:44 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:34:45.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:44 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:44 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3394381607' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:34:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:44 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:34:45.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:44 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:45.524 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-09-21T10:34:45.524 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-09-21T10:34:45.524 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-09-21T10:34:45.524 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-09-21T10:34:45.524 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T10:34:45.524 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:34:45.731 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:46.156 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[101299]: pgmap v1593: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:46.156 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:46.156 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:34:46.156 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[101299]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:34:46.156 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[101299]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T10:34:46.156 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:34:46.156 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:46.157 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[105748]: pgmap v1593: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:46.157 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:46.157 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:34:46.157 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[105748]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:34:46.157 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[105748]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T10:34:46.157 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:34:46.157 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:45 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:45 smithi045 ceph-mon[95976]: pgmap v1593: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:34:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:45 smithi045 ceph-mon[95976]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:34:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:45 smithi045 ceph-mon[95976]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T10:34:46.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:34:46.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:45 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' 2024-09-21T10:34:47.658 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:34:47.658 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":528,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:43:37.317307+0000","modified":"2024-09-21T10:34:44.851405+0000","last_up_change":"2024-09-21T10:33:12.904540+0000","last_in_change":"2024-09-21T10:33:35.989675+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:46:40.328691+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":21,"pg_placement_num":21,"pg_placement_num_target":21,"pg_num_target":21,"pg_num_pending":21,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"460","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"430","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8999999761581421,"score_stable":1.8999999761581421,"optimal_score":1,"raw_score_acting":1.8999999761581421,"raw_score_stable":1.8999999761581421,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:49:08.740859+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:50:21.329505+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:50:21.832696+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0f6f5dda-c134-4283-a91d-cf501ef19062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":389,"last_clean_end":454,"up_from":491,"up_thru":522,"down_at":455,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2741331695},{"type":"v1","addr":"172.21.15.32:6803","nonce":2741331695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2741331695},{"type":"v1","addr":"172.21.15.32:6805","nonce":2741331695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2741331695},{"type":"v1","addr":"172.21.15.32:6809","nonce":2741331695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2741331695},{"type":"v1","addr":"172.21.15.32:6807","nonce":2741331695}]},"public_addr":"172.21.15.32:6803/2741331695","cluster_addr":"172.21.15.32:6805/2741331695","heartbeat_back_addr":"172.21.15.32:6809/2741331695","heartbeat_front_addr":"172.21.15.32:6807/2741331695","state":["exists","up"]},{"osd":1,"uuid":"bdaa43fd-b969-4d06-b6eb-97b415c4e118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":522,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6811","nonce":1357153778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6813","nonce":1357153778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6817","nonce":1357153778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6815","nonce":1357153778}]},"public_addr":"172.21.15.32:6811/1357153778","cluster_addr":"172.21.15.32:6813/1357153778","heartbeat_back_addr":"172.21.15.32:6817/1357153778","heartbeat_front_addr":"172.21.15.32:6815/1357153778","state":["exists","up"]},{"osd":2,"uuid":"1f12d9f2-90b0-44f6-8432-0b398194b3b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":505,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6819","nonce":3097218490}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6821","nonce":3097218490}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6825","nonce":3097218490}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6823","nonce":3097218490}]},"public_addr":"172.21.15.32:6819/3097218490","cluster_addr":"172.21.15.32:6821/3097218490","heartbeat_back_addr":"172.21.15.32:6825/3097218490","heartbeat_front_addr":"172.21.15.32:6823/3097218490","state":["exists","up"]},{"osd":3,"uuid":"6a568fd5-5fd9-46f8-b3af-9fa196ed617b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":520,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6827","nonce":438066087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6829","nonce":438066087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6833","nonce":438066087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6831","nonce":438066087}]},"public_addr":"172.21.15.32:6827/438066087","cluster_addr":"172.21.15.32:6829/438066087","heartbeat_back_addr":"172.21.15.32:6833/438066087","heartbeat_front_addr":"172.21.15.32:6831/438066087","state":["exists","up"]},{"osd":4,"uuid":"2cf149e0-4c2f-46eb-9ca2-a51907aea3b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":518,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6801","nonce":413300252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6803","nonce":413300252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6807","nonce":413300252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6805","nonce":413300252}]},"public_addr":"172.21.15.45:6801/413300252","cluster_addr":"172.21.15.45:6803/413300252","heartbeat_back_addr":"172.21.15.45:6807/413300252","heartbeat_front_addr":"172.21.15.45:6805/413300252","state":["exists","up"]},{"osd":5,"uuid":"a6c78855-6756-4b0b-9a9c-8d102079540b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":280,"up_from":324,"up_thru":520,"down_at":281,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3383231394},{"type":"v1","addr":"172.21.15.45:6809","nonce":3383231394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":3383231394},{"type":"v1","addr":"172.21.15.45:6811","nonce":3383231394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":3383231394},{"type":"v1","addr":"172.21.15.45:6815","nonce":3383231394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":3383231394},{"type":"v1","addr":"172.21.15.45:6813","nonce":3383231394}]},"public_addr":"172.21.15.45:6809/3383231394","cluster_addr":"172.21.15.45:6811/3383231394","heartbeat_back_addr":"172.21.15.45:6815/3383231394","heartbeat_front_addr":"172.21.15.45:6813/3383231394","state":["exists","up"]},{"osd":6,"uuid":"d476169d-dffc-4f1e-b60b-4f836f53285e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":523,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6817","nonce":2158139679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6819","nonce":2158139679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6823","nonce":2158139679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6821","nonce":2158139679}]},"public_addr":"172.21.15.45:6817/2158139679","cluster_addr":"172.21.15.45:6819/2158139679","heartbeat_back_addr":"172.21.15.45:6823/2158139679","heartbeat_front_addr":"172.21.15.45:6821/2158139679","state":["exists","up"]},{"osd":7,"uuid":"021a4fd0-f770-4799-9683-b0aa7cd593bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":523,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6825","nonce":3336053350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6827","nonce":3336053350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6831","nonce":3336053350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6829","nonce":3336053350}]},"public_addr":"172.21.15.45:6825/3336053350","cluster_addr":"172.21.15.45:6827/3336053350","heartbeat_back_addr":"172.21.15.45:6831/3336053350","heartbeat_front_addr":"172.21.15.45:6829/3336053350","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:50.697928+0000","dead_epoch":454},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:13.831263+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:37.360357+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:00.010474+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:21.607129+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:45.583237+0000","dead_epoch":280},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:09.486296+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:33.456690+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":7}]},{"pgid":"4.1b","mappings":[{"from":1,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/3511996666":"2024-09-22T09:56:17.980985+0000","172.21.15.32:6800/2805220519":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1807347540":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1200491506":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2408427509":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1178553739":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6801/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.45:0/2114035806":"2024-09-22T09:53:38.533217+0000","172.21.15.32:6801/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.45:0/102537314":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/458122323":"2024-09-22T09:49:07.125921+0000","172.21.15.45:6835/1148645562":"2024-09-22T09:53:38.533217+0000","172.21.15.45:0/599831880":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/1465894634":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/230160407":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2785183325":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/431176591":"2024-09-22T09:53:08.870463+0000","172.21.15.32:6800/431176591":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2103196055":"2024-09-22T09:53:08.870463+0000","172.21.15.32:6801/2805220519":"2024-09-22T09:56:17.980985+0000","172.21.15.32:6800/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/2042031504":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/1259145858":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/533058733":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/1858508442":"2024-09-22T09:53:08.870463+0000","172.21.15.32:6801/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/2895073807":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.45:0/338006180":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/2388392475":"2024-09-22T09:49:07.125921+0000","172.21.15.45:6834/1148645562":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/1102447792":"2024-09-22T09:56:17.980985+0000","172.21.15.45:0/2100270146":"2024-09-22T09:53:38.533217+0000","172.21.15.45:0/431361432":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/3332221400":"2024-09-22T09:43:58.900171+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:34:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:47 smithi032 ceph-mon[101299]: pgmap v1595: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:48.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:47 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/4214294825' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:34:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:47 smithi032 ceph-mon[105748]: pgmap v1595: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:48.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:47 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/4214294825' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:34:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:47 smithi045 ceph-mon[95976]: pgmap v1595: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:48.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:47 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/4214294825' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:34:48.346 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T10:34:48.346 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:34:48.561 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:48.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:48 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:48.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:48 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:49.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:48 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:34:49.694 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:34:49 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:34:49] "GET /metrics HTTP/1.1" 200 34132 "" "Prometheus/2.51.0" 2024-09-21T10:34:49.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:49 smithi032 ceph-mon[101299]: pgmap v1596: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:49.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:49 smithi032 ceph-mon[105748]: pgmap v1596: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:50.186 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:49 smithi045 ceph-mon[95976]: pgmap v1596: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:50.421 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:34:50.421 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":528,"fsid":"e0bde4d4-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:43:37.317307+0000","modified":"2024-09-21T10:34:44.851405+0000","last_up_change":"2024-09-21T10:33:12.904540+0000","last_in_change":"2024-09-21T10:33:35.989675+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:46:40.328691+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":21,"pg_placement_num":21,"pg_placement_num_target":21,"pg_num_target":21,"pg_num_pending":21,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"460","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"430","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8999999761581421,"score_stable":1.8999999761581421,"optimal_score":1,"raw_score_acting":1.8999999761581421,"raw_score_stable":1.8999999761581421,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:49:08.740859+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:50:21.329505+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:50:21.832696+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0f6f5dda-c134-4283-a91d-cf501ef19062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":389,"last_clean_end":454,"up_from":491,"up_thru":522,"down_at":455,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2741331695},{"type":"v1","addr":"172.21.15.32:6803","nonce":2741331695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2741331695},{"type":"v1","addr":"172.21.15.32:6805","nonce":2741331695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2741331695},{"type":"v1","addr":"172.21.15.32:6809","nonce":2741331695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2741331695},{"type":"v1","addr":"172.21.15.32:6807","nonce":2741331695}]},"public_addr":"172.21.15.32:6803/2741331695","cluster_addr":"172.21.15.32:6805/2741331695","heartbeat_back_addr":"172.21.15.32:6809/2741331695","heartbeat_front_addr":"172.21.15.32:6807/2741331695","state":["exists","up"]},{"osd":1,"uuid":"bdaa43fd-b969-4d06-b6eb-97b415c4e118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":522,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6811","nonce":1357153778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6813","nonce":1357153778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6817","nonce":1357153778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":1357153778},{"type":"v1","addr":"172.21.15.32:6815","nonce":1357153778}]},"public_addr":"172.21.15.32:6811/1357153778","cluster_addr":"172.21.15.32:6813/1357153778","heartbeat_back_addr":"172.21.15.32:6817/1357153778","heartbeat_front_addr":"172.21.15.32:6815/1357153778","state":["exists","up"]},{"osd":2,"uuid":"1f12d9f2-90b0-44f6-8432-0b398194b3b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":505,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6819","nonce":3097218490}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6821","nonce":3097218490}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6825","nonce":3097218490}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3097218490},{"type":"v1","addr":"172.21.15.32:6823","nonce":3097218490}]},"public_addr":"172.21.15.32:6819/3097218490","cluster_addr":"172.21.15.32:6821/3097218490","heartbeat_back_addr":"172.21.15.32:6825/3097218490","heartbeat_front_addr":"172.21.15.32:6823/3097218490","state":["exists","up"]},{"osd":3,"uuid":"6a568fd5-5fd9-46f8-b3af-9fa196ed617b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":520,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6827","nonce":438066087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6829","nonce":438066087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6833","nonce":438066087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":438066087},{"type":"v1","addr":"172.21.15.32:6831","nonce":438066087}]},"public_addr":"172.21.15.32:6827/438066087","cluster_addr":"172.21.15.32:6829/438066087","heartbeat_back_addr":"172.21.15.32:6833/438066087","heartbeat_front_addr":"172.21.15.32:6831/438066087","state":["exists","up"]},{"osd":4,"uuid":"2cf149e0-4c2f-46eb-9ca2-a51907aea3b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":518,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6801","nonce":413300252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6803","nonce":413300252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6807","nonce":413300252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":413300252},{"type":"v1","addr":"172.21.15.45:6805","nonce":413300252}]},"public_addr":"172.21.15.45:6801/413300252","cluster_addr":"172.21.15.45:6803/413300252","heartbeat_back_addr":"172.21.15.45:6807/413300252","heartbeat_front_addr":"172.21.15.45:6805/413300252","state":["exists","up"]},{"osd":5,"uuid":"a6c78855-6756-4b0b-9a9c-8d102079540b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":280,"up_from":324,"up_thru":520,"down_at":281,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3383231394},{"type":"v1","addr":"172.21.15.45:6809","nonce":3383231394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":3383231394},{"type":"v1","addr":"172.21.15.45:6811","nonce":3383231394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":3383231394},{"type":"v1","addr":"172.21.15.45:6815","nonce":3383231394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":3383231394},{"type":"v1","addr":"172.21.15.45:6813","nonce":3383231394}]},"public_addr":"172.21.15.45:6809/3383231394","cluster_addr":"172.21.15.45:6811/3383231394","heartbeat_back_addr":"172.21.15.45:6815/3383231394","heartbeat_front_addr":"172.21.15.45:6813/3383231394","state":["exists","up"]},{"osd":6,"uuid":"d476169d-dffc-4f1e-b60b-4f836f53285e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":523,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6817","nonce":2158139679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6819","nonce":2158139679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6823","nonce":2158139679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2158139679},{"type":"v1","addr":"172.21.15.45:6821","nonce":2158139679}]},"public_addr":"172.21.15.45:6817/2158139679","cluster_addr":"172.21.15.45:6819/2158139679","heartbeat_back_addr":"172.21.15.45:6823/2158139679","heartbeat_front_addr":"172.21.15.45:6821/2158139679","state":["exists","up"]},{"osd":7,"uuid":"021a4fd0-f770-4799-9683-b0aa7cd593bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":523,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6825","nonce":3336053350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6827","nonce":3336053350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6831","nonce":3336053350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3336053350},{"type":"v1","addr":"172.21.15.45:6829","nonce":3336053350}]},"public_addr":"172.21.15.45:6825/3336053350","cluster_addr":"172.21.15.45:6827/3336053350","heartbeat_back_addr":"172.21.15.45:6831/3336053350","heartbeat_front_addr":"172.21.15.45:6829/3336053350","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:50.697928+0000","dead_epoch":454},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:13.831263+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:37.360357+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:00.010474+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:21.607129+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:47:45.583237+0000","dead_epoch":280},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:09.486296+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:48:33.456690+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":7}]},{"pgid":"4.1b","mappings":[{"from":1,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/3511996666":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1200491506":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/1465894634":"2024-09-22T09:53:08.870463+0000","172.21.15.32:6801/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.45:0/2114035806":"2024-09-22T09:53:38.533217+0000","172.21.15.32:6801/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.45:0/102537314":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/458122323":"2024-09-22T09:49:07.125921+0000","172.21.15.45:6835/1148645562":"2024-09-22T09:53:38.533217+0000","172.21.15.45:0/599831880":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/1807347540":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/230160407":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/4033152109":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/2785183325":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/431176591":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2408427509":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1178553739":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/431176591":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2103196055":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2042031504":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/1259145858":"2024-09-22T09:49:07.125921+0000","172.21.15.32:0/533058733":"2024-09-22T09:44:29.817113+0000","172.21.15.32:0/3332221400":"2024-09-22T09:43:58.900171+0000","172.21.15.45:0/2100270146":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/1102447792":"2024-09-22T09:56:17.980985+0000","172.21.15.45:0/431361432":"2024-09-22T09:53:38.533217+0000","172.21.15.32:6800/2033554920":"2024-09-22T09:49:07.125921+0000","172.21.15.32:6801/2805220519":"2024-09-22T09:56:17.980985+0000","172.21.15.32:0/1858508442":"2024-09-22T09:53:08.870463+0000","172.21.15.32:0/2895073807":"2024-09-22T09:44:29.817113+0000","172.21.15.32:6800/2819046815":"2024-09-22T09:43:58.900171+0000","172.21.15.32:6800/2805220519":"2024-09-22T09:56:17.980985+0000","172.21.15.45:0/338006180":"2024-09-22T09:53:38.533217+0000","172.21.15.32:0/2388392475":"2024-09-22T09:49:07.125921+0000","172.21.15.45:6834/1148645562":"2024-09-22T09:53:38.533217+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:34:51.119 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:50 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3120391675' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:34:51.119 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:50 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3120391675' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:34:51.145 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T10:34:51.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T10:34:51.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T10:34:51.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T10:34:51.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T10:34:51.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T10:34:51.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T10:34:51.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T10:34:51.202 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:50 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3120391675' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:34:51.610 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:51.612 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:51.613 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:51.613 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:51.617 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:51.617 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:51.627 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:51.635 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:34:52.028 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:51 smithi032 ceph-mon[101299]: pgmap v1597: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:52.028 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:51 smithi032 ceph-mon[105748]: pgmap v1597: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:52.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:51 smithi045 ceph-mon[95976]: pgmap v1597: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:54.089 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:53 smithi032 ceph-mon[105748]: pgmap v1598: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:54.090 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:53 smithi032 ceph-mon[101299]: pgmap v1598: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:54.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:53 smithi045 ceph-mon[95976]: pgmap v1598: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:56.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:55 smithi032 ceph-mon[101299]: pgmap v1599: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:56.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:55 smithi032 ceph-mon[105748]: pgmap v1599: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:56.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:55 smithi045 ceph-mon[95976]: pgmap v1599: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:57.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:57 smithi032 ceph-mon[101299]: pgmap v1600: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:57.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:57 smithi032 ceph-mon[105748]: pgmap v1600: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:58.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:57 smithi045 ceph-mon[95976]: pgmap v1600: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:34:59.827 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:34:59 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:34:59] "GET /metrics HTTP/1.1" 200 34132 "" "Prometheus/2.51.0" 2024-09-21T10:35:00.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:34:59 smithi032 ceph-mon[101299]: pgmap v1601: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:00.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:34:59 smithi032 ceph-mon[105748]: pgmap v1601: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:00.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:34:59 smithi045 ceph-mon[95976]: pgmap v1601: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:01.250 INFO:teuthology.orchestra.run.smithi032.stdout:425201762694 2024-09-21T10:35:01.251 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T10:35:02.056 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:01 smithi032 ceph-mon[101299]: pgmap v1602: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:02.056 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:01 smithi032 ceph-mon[105748]: pgmap v1602: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:02.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:01 smithi045 ceph-mon[95976]: pgmap v1602: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:03.816 INFO:teuthology.orchestra.run.smithi032.stdout:468151435640 2024-09-21T10:35:03.816 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T10:35:03.950 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:03 smithi032 ceph-mon[105748]: pgmap v1603: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:03.950 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:03 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:35:03.950 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:03 smithi032 ceph-mon[101299]: pgmap v1603: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:03.950 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:03 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:35:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:03 smithi045 ceph-mon[95976]: pgmap v1603: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:04.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:03 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:35:06.003 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:05 smithi032 ceph-mon[101299]: pgmap v1604: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:06.003 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:05 smithi032 ceph-mon[105748]: pgmap v1604: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:06.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:05 smithi045 ceph-mon[95976]: pgmap v1604: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:07.161 INFO:teuthology.orchestra.run.smithi032.stdout:231928234545 2024-09-21T10:35:07.161 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T10:35:08.006 INFO:teuthology.orchestra.run.smithi032.stdout:446676599168 2024-09-21T10:35:08.007 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T10:35:08.190 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:07 smithi032 ceph-mon[105748]: pgmap v1605: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:08.191 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:07 smithi032 ceph-mon[101299]: pgmap v1605: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:08.198 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:08.198 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:08.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:07 smithi045 ceph-mon[95976]: pgmap v1605: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:08.980 INFO:teuthology.orchestra.run.smithi032.stdout:2108828942360 2024-09-21T10:35:08.980 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T10:35:09.641 INFO:teuthology.orchestra.run.smithi032.stdout:1391569404084 2024-09-21T10:35:09.641 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T10:35:09.853 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:35:09] "GET /metrics HTTP/1.1" 200 34132 "" "Prometheus/2.51.0" 2024-09-21T10:35:10.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:09 smithi032 ceph-mon[101299]: pgmap v1606: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:10.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:09 smithi032 ceph-mon[105748]: pgmap v1606: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:10.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:09 smithi045 ceph-mon[95976]: pgmap v1606: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:12.126 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:11 smithi032 ceph-mon[101299]: pgmap v1607: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:12.126 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:11 smithi032 ceph-mon[105748]: pgmap v1607: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:12.154 INFO:teuthology.orchestra.run.smithi032.stdout:128849019458 2024-09-21T10:35:12.155 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T10:35:12.157 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:12.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:11 smithi045 ceph-mon[95976]: pgmap v1607: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:12.627 INFO:teuthology.orchestra.run.smithi032.stdout:197568496183 2024-09-21T10:35:12.627 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T10:35:13.378 INFO:teuthology.orchestra.run.smithi032.stdout:425201762696 2024-09-21T10:35:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:13 smithi032 ceph-mon[105748]: pgmap v1608: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:14.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:13 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1047433974' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:35:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:13 smithi032 ceph-mon[101299]: pgmap v1608: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:14.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:13 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1047433974' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:35:14.224 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:13 smithi045 ceph-mon[95976]: pgmap v1608: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:14.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:13 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1047433974' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:35:14.872 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762694 got 425201762696 for osd.1 2024-09-21T10:35:14.872 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:15.956 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:15 smithi032 ceph-mon[101299]: pgmap v1609: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:15.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:15 smithi032 ceph-mon[105748]: pgmap v1609: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:16.140 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:16.140 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:16.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:15 smithi045 ceph-mon[95976]: pgmap v1609: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:17.957 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:17 smithi032 ceph-mon[105748]: pgmap v1610: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:17.957 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:17 smithi032 ceph-mon[101299]: pgmap v1610: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:18.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:17 smithi045 ceph-mon[95976]: pgmap v1610: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:19.017 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:18 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:35:19.017 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:18 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:35:19.217 INFO:teuthology.orchestra.run.smithi032.stdout:446676599171 2024-09-21T10:35:19.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:18 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:35:19.417 INFO:teuthology.orchestra.run.smithi032.stdout:231928234548 2024-09-21T10:35:19.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:19 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:35:19] "GET /metrics HTTP/1.1" 200 34136 "" "Prometheus/2.51.0" 2024-09-21T10:35:20.149 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:19 smithi032 ceph-mon[105748]: pgmap v1611: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:20.149 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1839678992' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:35:20.149 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:19 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1692700875' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:35:20.150 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:19 smithi032 ceph-mon[101299]: pgmap v1611: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:20.150 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1839678992' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:35:20.150 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:19 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1692700875' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:35:20.173 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676599168 got 446676599171 for osd.2 2024-09-21T10:35:20.173 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:20.186 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:19 smithi045 ceph-mon[95976]: pgmap v1611: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:20.186 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1839678992' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:35:20.187 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:19 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1692700875' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:35:20.829 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234545 got 231928234548 for osd.7 2024-09-21T10:35:20.829 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:21.524 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:21.525 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:21.731 INFO:teuthology.orchestra.run.smithi032.stdout:468151435644 2024-09-21T10:35:22.052 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:21 smithi032 ceph-mon[105748]: pgmap v1612: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:22.052 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:21 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2181283373' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:35:22.053 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:21 smithi032 ceph-mon[101299]: pgmap v1612: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:22.053 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:21 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2181283373' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:35:22.259 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:21 smithi045 ceph-mon[95976]: pgmap v1612: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:22.259 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:21 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2181283373' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:35:22.259 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:35:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=cleanup t=2024-09-21T10:35:22.126733794Z level=info msg="Completed cleanup jobs" duration=2.951079ms 2024-09-21T10:35:22.535 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:35:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=plugins.update.checker t=2024-09-21T10:35:22.25748992Z level=info msg="Update check succeeded" duration=59.946469ms 2024-09-21T10:35:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:24 smithi032 ceph-mon[101299]: pgmap v1613: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:24.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:24 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/3986834892' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:35:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:24 smithi032 ceph-mon[105748]: pgmap v1613: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:24.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:24 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/3986834892' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:35:24.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:24 smithi045 ceph-mon[95976]: pgmap v1613: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:24.286 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:24 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/3986834892' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:35:24.464 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435640 got 468151435644 for osd.3 2024-09-21T10:35:24.464 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:25.624 INFO:teuthology.orchestra.run.smithi032.stdout:1391569404087 2024-09-21T10:35:26.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:26 smithi032 ceph-mon[105748]: pgmap v1614: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:26.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:26 smithi032 ceph-mon[101299]: pgmap v1614: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:26.237 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1391569404084 got 1391569404087 for osd.5 2024-09-21T10:35:26.237 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:26.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:26 smithi045 ceph-mon[95976]: pgmap v1614: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:26.772 INFO:teuthology.orchestra.run.smithi032.stdout:128849019461 2024-09-21T10:35:27.118 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:27 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/1635877724' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:35:27.118 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:27 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/1635877724' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:35:27.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:27 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/1635877724' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:35:27.400 INFO:teuthology.orchestra.run.smithi032.stdout:2108828942364 2024-09-21T10:35:28.107 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:28 smithi032 ceph-mon[105748]: pgmap v1615: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:28.107 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:28 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/2914937908' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:35:28.108 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:28 smithi032 ceph-mon[101299]: pgmap v1615: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:28.108 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:28 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/2914937908' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:35:28.133 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2108828942360 got 2108828942364 for osd.0 2024-09-21T10:35:28.134 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:28 smithi045 ceph-mon[95976]: pgmap v1615: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:28.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:28 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/2914937908' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:35:28.671 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019458 got 128849019461 for osd.4 2024-09-21T10:35:28.671 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:29.100 INFO:teuthology.orchestra.run.smithi032.stdout:197568496188 2024-09-21T10:35:29.875 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:29 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:35:29] "GET /metrics HTTP/1.1" 200 34136 "" "Prometheus/2.51.0" 2024-09-21T10:35:29.900 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496183 got 197568496188 for osd.6 2024-09-21T10:35:29.900 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:29.900 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:35:29.901 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:35:30.115 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:30.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:30 smithi032 ceph-mon[101299]: pgmap v1616: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:30.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:30 smithi032 ceph-mon[101299]: from='client.? 172.21.15.32:0/746994604' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:35:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:30 smithi032 ceph-mon[105748]: pgmap v1616: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:30.207 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:30 smithi032 ceph-mon[105748]: from='client.? 172.21.15.32:0/746994604' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:35:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:30 smithi045 ceph-mon[95976]: pgmap v1616: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:30.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:30 smithi045 ceph-mon[95976]: from='client.? 172.21.15.32:0/746994604' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:35:32.053 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:35:32.054 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:35:32.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:32 smithi045 ceph-mon[95976]: pgmap v1617: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:32.341 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:32 smithi032 ceph-mon[105748]: pgmap v1617: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:32.341 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:32 smithi032 ceph-mon[101299]: pgmap v1617: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:33.057 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1617,"stamp":"2024-09-21T10:35:30.811156+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":908,"num_write_kb":12625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":461,"num_bytes_recovered":34544173,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4166,"ondisk_log_size":4166,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6524812,"kb_used_data":66924,"kb_used_omap":49,"kb_used_meta":6457742,"kb_avail":743436404,"statfs":{"total":767960285184,"available":761278877696,"internally_reserved":0,"allocated":68530176,"data_stored":47709999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":50491,"internal_metadata":6612728517},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003105"},"pg_stats":[{"pgid":"1.11","version":"90'149","reported_seq":779,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937319+0000","last_change":"2024-09-21T10:33:39.997985+0000","last_active":"2024-09-21T10:34:44.937319+0000","last_peered":"2024-09-21T10:34:44.937319+0000","last_clean":"2024-09-21T10:34:44.937319+0000","last_became_active":"2024-09-21T10:33:39.997631+0000","last_became_peered":"2024-09-21T10:33:39.997631+0000","last_unstale":"2024-09-21T10:34:44.937319+0000","last_undegraded":"2024-09-21T10:34:44.937319+0000","last_fullsized":"2024-09-21T10:34:44.937319+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:26.643960+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:30:26.643960+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:24.384086+0000","scrub_duration":0.013444612,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":952,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857669+0000","last_change":"2024-09-21T10:30:45.210213+0000","last_active":"2024-09-21T10:34:44.857669+0000","last_peered":"2024-09-21T10:34:44.857669+0000","last_clean":"2024-09-21T10:34:44.857669+0000","last_became_active":"2024-09-21T10:30:45.207807+0000","last_became_peered":"2024-09-21T10:30:45.207807+0000","last_unstale":"2024-09-21T10:34:44.857669+0000","last_undegraded":"2024-09-21T10:34:44.857669+0000","last_fullsized":"2024-09-21T10:34:44.857669+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:00.839446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":536,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.526423+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:34:44.526423+0000","last_peered":"2024-09-21T10:34:44.526423+0000","last_clean":"2024-09-21T10:34:44.526423+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:34:44.526423+0000","last_undegraded":"2024-09-21T10:34:44.526423+0000","last_fullsized":"2024-09-21T10:34:44.526423+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'149","reported_seq":707,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857446+0000","last_change":"2024-09-21T10:30:58.557091+0000","last_active":"2024-09-21T10:34:44.857446+0000","last_peered":"2024-09-21T10:34:44.857446+0000","last_clean":"2024-09-21T10:34:44.857446+0000","last_became_active":"2024-09-21T10:30:31.102167+0000","last_became_peered":"2024-09-21T10:30:31.102167+0000","last_unstale":"2024-09-21T10:34:44.857446+0000","last_undegraded":"2024-09-21T10:34:44.857446+0000","last_fullsized":"2024-09-21T10:34:44.857446+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:58.556988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:58.556988+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:55.249909+0000","scrub_duration":0.0013450059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":232,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.353692+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:34:41.353692+0000","last_peered":"2024-09-21T10:34:41.353692+0000","last_clean":"2024-09-21T10:34:41.353692+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:34:41.353692+0000","last_undegraded":"2024-09-21T10:34:41.353692+0000","last_fullsized":"2024-09-21T10:34:41.353692+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":69,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937024+0000","last_change":"2024-09-21T10:34:26.160198+0000","last_active":"2024-09-21T10:34:44.937024+0000","last_peered":"2024-09-21T10:34:44.937024+0000","last_clean":"2024-09-21T10:34:44.937024+0000","last_became_active":"2024-09-21T10:34:26.159818+0000","last_became_peered":"2024-09-21T10:34:26.159818+0000","last_unstale":"2024-09-21T10:34:44.937024+0000","last_undegraded":"2024-09-21T10:34:44.937024+0000","last_fullsized":"2024-09-21T10:34:44.937024+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:53.948099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"90'149","reported_seq":920,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937736+0000","last_change":"2024-09-21T10:30:47.984484+0000","last_active":"2024-09-21T10:34:44.937736+0000","last_peered":"2024-09-21T10:34:44.937736+0000","last_clean":"2024-09-21T10:34:44.937736+0000","last_became_active":"2024-09-21T10:30:47.981723+0000","last_became_peered":"2024-09-21T10:30:47.981723+0000","last_unstale":"2024-09-21T10:34:44.937736+0000","last_undegraded":"2024-09-21T10:34:44.937736+0000","last_fullsized":"2024-09-21T10:34:44.937736+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:26.687488+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:30:26.687488+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:17.623163+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":69,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937516+0000","last_change":"2024-09-21T10:34:26.161153+0000","last_active":"2024-09-21T10:34:44.937516+0000","last_peered":"2024-09-21T10:34:44.937516+0000","last_clean":"2024-09-21T10:34:44.937516+0000","last_became_active":"2024-09-21T10:34:26.160837+0000","last_became_peered":"2024-09-21T10:34:26.160837+0000","last_unstale":"2024-09-21T10:34:44.937516+0000","last_undegraded":"2024-09-21T10:34:44.937516+0000","last_fullsized":"2024-09-21T10:34:44.937516+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:09.108139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":507,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.526389+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:34:44.526389+0000","last_peered":"2024-09-21T10:34:44.526389+0000","last_clean":"2024-09-21T10:34:44.526389+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:34:44.526389+0000","last_undegraded":"2024-09-21T10:34:44.526389+0000","last_fullsized":"2024-09-21T10:34:44.526389+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'149","reported_seq":741,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857925+0000","last_change":"2024-09-21T10:30:59.599305+0000","last_active":"2024-09-21T10:34:44.857925+0000","last_peered":"2024-09-21T10:34:44.857925+0000","last_clean":"2024-09-21T10:34:44.857925+0000","last_became_active":"2024-09-21T10:30:47.960200+0000","last_became_peered":"2024-09-21T10:30:47.960200+0000","last_unstale":"2024-09-21T10:34:44.857925+0000","last_undegraded":"2024-09-21T10:34:44.857925+0000","last_fullsized":"2024-09-21T10:34:44.857925+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:59.599224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:59.599224+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:54.053533+0000","scrub_duration":0.001301359,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":987,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857878+0000","last_change":"2024-09-21T10:34:26.159055+0000","last_active":"2024-09-21T10:34:44.857878+0000","last_peered":"2024-09-21T10:34:44.857878+0000","last_clean":"2024-09-21T10:34:44.857878+0000","last_became_active":"2024-09-21T10:34:26.158706+0000","last_became_peered":"2024-09-21T10:34:26.158706+0000","last_unstale":"2024-09-21T10:34:44.857878+0000","last_undegraded":"2024-09-21T10:34:44.857878+0000","last_fullsized":"2024-09-21T10:34:44.857878+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:11.025996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":647,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906164+0000","last_change":"2024-09-21T10:33:37.006914+0000","last_active":"2024-09-21T10:34:40.906164+0000","last_peered":"2024-09-21T10:34:40.906164+0000","last_clean":"2024-09-21T10:34:40.906164+0000","last_became_active":"2024-09-21T10:33:36.994021+0000","last_became_peered":"2024-09-21T10:33:36.994021+0000","last_unstale":"2024-09-21T10:34:40.906164+0000","last_undegraded":"2024-09-21T10:34:40.906164+0000","last_fullsized":"2024-09-21T10:34:40.906164+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:25.986510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'149","reported_seq":661,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.478719+0000","last_change":"2024-09-21T10:34:32.934865+0000","last_active":"2024-09-21T10:34:40.478719+0000","last_peered":"2024-09-21T10:34:40.478719+0000","last_clean":"2024-09-21T10:34:40.478719+0000","last_became_active":"2024-09-21T10:34:29.178213+0000","last_became_peered":"2024-09-21T10:34:29.178213+0000","last_unstale":"2024-09-21T10:34:40.478719+0000","last_undegraded":"2024-09-21T10:34:40.478719+0000","last_fullsized":"2024-09-21T10:34:40.478719+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:34:32.934773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:34:32.934773+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:06.954855+0000","scrub_duration":0.0014781499999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":22,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.478805+0000","last_change":"2024-09-21T10:34:26.166495+0000","last_active":"2024-09-21T10:34:40.478805+0000","last_peered":"2024-09-21T10:34:40.478805+0000","last_clean":"2024-09-21T10:34:40.478805+0000","last_became_active":"2024-09-21T10:34:26.160915+0000","last_became_peered":"2024-09-21T10:34:26.160915+0000","last_unstale":"2024-09-21T10:34:40.478805+0000","last_undegraded":"2024-09-21T10:34:40.478805+0000","last_fullsized":"2024-09-21T10:34:40.478805+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:39:08.788925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":259,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857647+0000","last_change":"2024-09-21T10:34:26.154060+0000","last_active":"2024-09-21T10:34:44.857647+0000","last_peered":"2024-09-21T10:34:44.857647+0000","last_clean":"2024-09-21T10:34:44.857647+0000","last_became_active":"2024-09-21T10:34:26.153721+0000","last_became_peered":"2024-09-21T10:34:26.153721+0000","last_unstale":"2024-09-21T10:34:44.857647+0000","last_undegraded":"2024-09-21T10:34:44.857647+0000","last_fullsized":"2024-09-21T10:34:44.857647+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:21.988421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"373'150","reported_seq":1016,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857551+0000","last_change":"2024-09-21T10:33:39.996646+0000","last_active":"2024-09-21T10:34:44.857551+0000","last_peered":"2024-09-21T10:34:44.857551+0000","last_clean":"2024-09-21T10:34:44.857551+0000","last_became_active":"2024-09-21T10:33:39.996222+0000","last_became_peered":"2024-09-21T10:33:39.996222+0000","last_unstale":"2024-09-21T10:34:44.857551+0000","last_undegraded":"2024-09-21T10:34:44.857551+0000","last_fullsized":"2024-09-21T10:34:44.857551+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":4,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:30:24.812285+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:30:24.812285+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:40.967138+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":236,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938322+0000","last_change":"2024-09-21T10:30:45.225017+0000","last_active":"2024-09-21T10:34:44.938322+0000","last_peered":"2024-09-21T10:34:44.938322+0000","last_clean":"2024-09-21T10:34:44.938322+0000","last_became_active":"2024-09-21T10:30:45.214625+0000","last_became_peered":"2024-09-21T10:30:45.214625+0000","last_unstale":"2024-09-21T10:34:44.938322+0000","last_undegraded":"2024-09-21T10:34:44.938322+0000","last_fullsized":"2024-09-21T10:34:44.938322+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:41.994673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":247,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857170+0000","last_change":"2024-09-21T10:30:45.206472+0000","last_active":"2024-09-21T10:34:44.857170+0000","last_peered":"2024-09-21T10:34:44.857170+0000","last_clean":"2024-09-21T10:34:44.857170+0000","last_became_active":"2024-09-21T10:30:45.205955+0000","last_became_peered":"2024-09-21T10:30:45.205955+0000","last_unstale":"2024-09-21T10:34:44.857170+0000","last_undegraded":"2024-09-21T10:34:44.857170+0000","last_fullsized":"2024-09-21T10:34:44.857170+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:50.262089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'149","reported_seq":653,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938285+0000","last_change":"2024-09-21T10:31:01.909114+0000","last_active":"2024-09-21T10:34:44.938285+0000","last_peered":"2024-09-21T10:34:44.938285+0000","last_clean":"2024-09-21T10:34:44.938285+0000","last_became_active":"2024-09-21T10:30:31.109278+0000","last_became_peered":"2024-09-21T10:30:31.109278+0000","last_unstale":"2024-09-21T10:34:44.938285+0000","last_undegraded":"2024-09-21T10:34:44.938285+0000","last_fullsized":"2024-09-21T10:34:44.938285+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:31:01.905848+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:31:01.905848+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:16:19.880231+0000","scrub_duration":0.018313652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":532,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938126+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:34:44.938126+0000","last_peered":"2024-09-21T10:34:44.938126+0000","last_clean":"2024-09-21T10:34:44.938126+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:34:44.938126+0000","last_undegraded":"2024-09-21T10:34:44.938126+0000","last_fullsized":"2024-09-21T10:34:44.938126+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":587,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906271+0000","last_change":"2024-09-21T10:34:26.157998+0000","last_active":"2024-09-21T10:34:40.906271+0000","last_peered":"2024-09-21T10:34:40.906271+0000","last_clean":"2024-09-21T10:34:40.906271+0000","last_became_active":"2024-09-21T10:34:26.157665+0000","last_became_peered":"2024-09-21T10:34:26.157665+0000","last_unstale":"2024-09-21T10:34:40.906271+0000","last_undegraded":"2024-09-21T10:34:40.906271+0000","last_fullsized":"2024-09-21T10:34:40.906271+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:47:06.546254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'149","reported_seq":707,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857736+0000","last_change":"2024-09-21T10:31:00.501007+0000","last_active":"2024-09-21T10:34:44.857736+0000","last_peered":"2024-09-21T10:34:44.857736+0000","last_clean":"2024-09-21T10:34:44.857736+0000","last_became_active":"2024-09-21T10:30:31.105952+0000","last_became_peered":"2024-09-21T10:30:31.105952+0000","last_unstale":"2024-09-21T10:34:44.857736+0000","last_undegraded":"2024-09-21T10:34:44.857736+0000","last_fullsized":"2024-09-21T10:34:44.857736+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:31:00.500754+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:31:00.500754+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:00.821304+0000","scrub_duration":0.0048757180000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":465,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.478163+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:34:40.478163+0000","last_peered":"2024-09-21T10:34:40.478163+0000","last_clean":"2024-09-21T10:34:40.478163+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:34:40.478163+0000","last_undegraded":"2024-09-21T10:34:40.478163+0000","last_fullsized":"2024-09-21T10:34:40.478163+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937313+0000","last_change":"2024-09-21T10:33:37.001969+0000","last_active":"2024-09-21T10:34:44.937313+0000","last_peered":"2024-09-21T10:34:44.937313+0000","last_clean":"2024-09-21T10:34:44.937313+0000","last_became_active":"2024-09-21T10:33:37.000831+0000","last_became_peered":"2024-09-21T10:33:37.000831+0000","last_unstale":"2024-09-21T10:34:44.937313+0000","last_undegraded":"2024-09-21T10:34:44.937313+0000","last_fullsized":"2024-09-21T10:34:44.937313+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:17.652214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":817,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.477685+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:34:40.477685+0000","last_peered":"2024-09-21T10:34:40.477685+0000","last_clean":"2024-09-21T10:34:40.477685+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:34:40.477685+0000","last_undegraded":"2024-09-21T10:34:40.477685+0000","last_fullsized":"2024-09-21T10:34:40.477685+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1026,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857607+0000","last_change":"2024-09-21T10:33:37.018129+0000","last_active":"2024-09-21T10:34:44.857607+0000","last_peered":"2024-09-21T10:34:44.857607+0000","last_clean":"2024-09-21T10:34:44.857607+0000","last_became_active":"2024-09-21T10:33:36.999471+0000","last_became_peered":"2024-09-21T10:33:36.999471+0000","last_unstale":"2024-09-21T10:34:44.857607+0000","last_undegraded":"2024-09-21T10:34:44.857607+0000","last_fullsized":"2024-09-21T10:34:44.857607+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:01.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":228,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226587+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:34:41.226587+0000","last_peered":"2024-09-21T10:34:41.226587+0000","last_clean":"2024-09-21T10:34:41.226587+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:34:41.226587+0000","last_undegraded":"2024-09-21T10:34:41.226587+0000","last_fullsized":"2024-09-21T10:34:41.226587+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":590,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857556+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:34:44.857556+0000","last_peered":"2024-09-21T10:34:44.857556+0000","last_clean":"2024-09-21T10:34:44.857556+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:34:44.857556+0000","last_undegraded":"2024-09-21T10:34:44.857556+0000","last_fullsized":"2024-09-21T10:34:44.857556+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":559,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906406+0000","last_change":"2024-09-21T10:30:45.223044+0000","last_active":"2024-09-21T10:34:40.906406+0000","last_peered":"2024-09-21T10:34:40.906406+0000","last_clean":"2024-09-21T10:34:40.906406+0000","last_became_active":"2024-09-21T10:30:45.222166+0000","last_became_peered":"2024-09-21T10:30:45.222166+0000","last_unstale":"2024-09-21T10:34:40.906406+0000","last_undegraded":"2024-09-21T10:34:40.906406+0000","last_fullsized":"2024-09-21T10:34:40.906406+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:16.215392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":246,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226929+0000","last_change":"2024-09-21T10:30:45.206465+0000","last_active":"2024-09-21T10:34:41.226929+0000","last_peered":"2024-09-21T10:34:41.226929+0000","last_clean":"2024-09-21T10:34:41.226929+0000","last_became_active":"2024-09-21T10:30:45.206088+0000","last_became_peered":"2024-09-21T10:30:45.206088+0000","last_unstale":"2024-09-21T10:34:41.226929+0000","last_undegraded":"2024-09-21T10:34:41.226929+0000","last_fullsized":"2024-09-21T10:34:41.226929+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:46:01.557325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'149","reported_seq":965,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938096+0000","last_change":"2024-09-21T10:33:40.003862+0000","last_active":"2024-09-21T10:34:44.938096+0000","last_peered":"2024-09-21T10:34:44.938096+0000","last_clean":"2024-09-21T10:34:44.938096+0000","last_became_active":"2024-09-21T10:33:40.003416+0000","last_became_peered":"2024-09-21T10:33:40.003416+0000","last_unstale":"2024-09-21T10:34:44.938096+0000","last_undegraded":"2024-09-21T10:34:44.938096+0000","last_fullsized":"2024-09-21T10:34:44.938096+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:24.704596+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:30:24.704596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:56:28.377638+0000","scrub_duration":14,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":229,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857041+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:34:44.857041+0000","last_peered":"2024-09-21T10:34:44.857041+0000","last_clean":"2024-09-21T10:34:44.857041+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:34:44.857041+0000","last_undegraded":"2024-09-21T10:34:44.857041+0000","last_fullsized":"2024-09-21T10:34:44.857041+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":246,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226998+0000","last_change":"2024-09-21T10:30:45.220586+0000","last_active":"2024-09-21T10:34:41.226998+0000","last_peered":"2024-09-21T10:34:41.226998+0000","last_clean":"2024-09-21T10:34:41.226998+0000","last_became_active":"2024-09-21T10:30:45.219727+0000","last_became_peered":"2024-09-21T10:30:45.219727+0000","last_unstale":"2024-09-21T10:34:41.226998+0000","last_undegraded":"2024-09-21T10:34:41.226998+0000","last_fullsized":"2024-09-21T10:34:41.226998+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:42.454658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":768,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857830+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:34:44.857830+0000","last_peered":"2024-09-21T10:34:44.857830+0000","last_clean":"2024-09-21T10:34:44.857830+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:34:44.857830+0000","last_undegraded":"2024-09-21T10:34:44.857830+0000","last_fullsized":"2024-09-21T10:34:44.857830+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":981,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857109+0000","last_change":"2024-09-21T10:30:45.207431+0000","last_active":"2024-09-21T10:34:44.857109+0000","last_peered":"2024-09-21T10:34:44.857109+0000","last_clean":"2024-09-21T10:34:44.857109+0000","last_became_active":"2024-09-21T10:30:45.207075+0000","last_became_peered":"2024-09-21T10:30:45.207075+0000","last_unstale":"2024-09-21T10:34:44.857109+0000","last_undegraded":"2024-09-21T10:34:44.857109+0000","last_fullsized":"2024-09-21T10:34:44.857109+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:08.684578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":80,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938382+0000","last_change":"2024-09-21T10:33:21.933455+0000","last_active":"2024-09-21T10:34:44.938382+0000","last_peered":"2024-09-21T10:34:44.938382+0000","last_clean":"2024-09-21T10:34:44.938382+0000","last_became_active":"2024-09-21T10:33:21.933188+0000","last_became_peered":"2024-09-21T10:33:21.933188+0000","last_unstale":"2024-09-21T10:34:44.938382+0000","last_undegraded":"2024-09-21T10:34:44.938382+0000","last_fullsized":"2024-09-21T10:34:44.938382+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:38.403373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":838,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.477608+0000","last_change":"2024-09-21T10:34:29.176105+0000","last_active":"2024-09-21T10:34:40.477608+0000","last_peered":"2024-09-21T10:34:40.477608+0000","last_clean":"2024-09-21T10:34:40.477608+0000","last_became_active":"2024-09-21T10:34:29.175632+0000","last_became_peered":"2024-09-21T10:34:29.175632+0000","last_unstale":"2024-09-21T10:34:40.477608+0000","last_undegraded":"2024-09-21T10:34:40.477608+0000","last_fullsized":"2024-09-21T10:34:40.477608+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:32:45.775219+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:32:45.775219+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:15.888314+0000","scrub_duration":0.016223530999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":845,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858390+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:34:44.858390+0000","last_peered":"2024-09-21T10:34:44.858390+0000","last_clean":"2024-09-21T10:34:44.858390+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:34:44.858390+0000","last_undegraded":"2024-09-21T10:34:44.858390+0000","last_fullsized":"2024-09-21T10:34:44.858390+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":899,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857381+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:34:44.857381+0000","last_peered":"2024-09-21T10:34:44.857381+0000","last_clean":"2024-09-21T10:34:44.857381+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:34:44.857381+0000","last_undegraded":"2024-09-21T10:34:44.857381+0000","last_fullsized":"2024-09-21T10:34:44.857381+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":508,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906578+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:34:40.906578+0000","last_peered":"2024-09-21T10:34:40.906578+0000","last_clean":"2024-09-21T10:34:40.906578+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:34:40.906578+0000","last_undegraded":"2024-09-21T10:34:40.906578+0000","last_fullsized":"2024-09-21T10:34:40.906578+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":777,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938037+0000","last_change":"2024-09-21T10:30:58.540050+0000","last_active":"2024-09-21T10:34:44.938037+0000","last_peered":"2024-09-21T10:34:44.938037+0000","last_clean":"2024-09-21T10:34:44.938037+0000","last_became_active":"2024-09-21T10:30:47.975931+0000","last_became_peered":"2024-09-21T10:30:47.975931+0000","last_unstale":"2024-09-21T10:34:44.938037+0000","last_undegraded":"2024-09-21T10:34:44.938037+0000","last_fullsized":"2024-09-21T10:34:44.938037+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:58.539975+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:30:58.539975+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:02.759105+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1367,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937393+0000","last_change":"2024-09-21T10:33:39.998744+0000","last_active":"2024-09-21T10:34:44.937393+0000","last_peered":"2024-09-21T10:34:44.937393+0000","last_clean":"2024-09-21T10:34:44.937393+0000","last_became_active":"2024-09-21T10:33:39.998073+0000","last_became_peered":"2024-09-21T10:33:39.998073+0000","last_unstale":"2024-09-21T10:34:44.937393+0000","last_undegraded":"2024-09-21T10:34:44.937393+0000","last_fullsized":"2024-09-21T10:34:44.937393+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:17.802386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":33554432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937005+0000","last_change":"2024-09-21T10:33:37.004358+0000","last_active":"2024-09-21T10:34:44.937005+0000","last_peered":"2024-09-21T10:34:44.937005+0000","last_clean":"2024-09-21T10:34:44.937005+0000","last_became_active":"2024-09-21T10:33:37.003994+0000","last_became_peered":"2024-09-21T10:33:37.003994+0000","last_unstale":"2024-09-21T10:34:44.937005+0000","last_undegraded":"2024-09-21T10:34:44.937005+0000","last_fullsized":"2024-09-21T10:34:44.937005+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:43.298113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937293+0000","last_change":"2024-09-21T10:33:36.996808+0000","last_active":"2024-09-21T10:34:44.937293+0000","last_peered":"2024-09-21T10:34:44.937293+0000","last_clean":"2024-09-21T10:34:44.937293+0000","last_became_active":"2024-09-21T10:33:36.996451+0000","last_became_peered":"2024-09-21T10:33:36.996451+0000","last_unstale":"2024-09-21T10:34:44.937293+0000","last_undegraded":"2024-09-21T10:34:44.937293+0000","last_fullsized":"2024-09-21T10:34:44.937293+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:28:01.658066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":855,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.477839+0000","last_change":"2024-09-21T10:34:29.178210+0000","last_active":"2024-09-21T10:34:40.477839+0000","last_peered":"2024-09-21T10:34:40.477839+0000","last_clean":"2024-09-21T10:34:40.477839+0000","last_became_active":"2024-09-21T10:34:29.177782+0000","last_became_peered":"2024-09-21T10:34:29.177782+0000","last_unstale":"2024-09-21T10:34:40.477839+0000","last_undegraded":"2024-09-21T10:34:40.477839+0000","last_fullsized":"2024-09-21T10:34:40.477839+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:32:53.227153+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:32:53.227153+0000","last_clean_scrub_stamp":"2024-09-21T10:32:53.227153+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:24:21.865898+0000","scrub_duration":0.010887529,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1567,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937395+0000","last_change":"2024-09-21T10:33:39.994395+0000","last_active":"2024-09-21T10:34:44.937395+0000","last_peered":"2024-09-21T10:34:44.937395+0000","last_clean":"2024-09-21T10:34:44.937395+0000","last_became_active":"2024-09-21T10:33:39.993621+0000","last_became_peered":"2024-09-21T10:33:39.993621+0000","last_unstale":"2024-09-21T10:34:44.937395+0000","last_undegraded":"2024-09-21T10:34:44.937395+0000","last_fullsized":"2024-09-21T10:34:44.937395+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:22.740479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937245+0000","last_change":"2024-09-21T10:30:45.211483+0000","last_active":"2024-09-21T10:34:44.937245+0000","last_peered":"2024-09-21T10:34:44.937245+0000","last_clean":"2024-09-21T10:34:44.937245+0000","last_became_active":"2024-09-21T10:30:45.211153+0000","last_became_peered":"2024-09-21T10:30:45.211153+0000","last_unstale":"2024-09-21T10:34:44.937245+0000","last_undegraded":"2024-09-21T10:34:44.937245+0000","last_fullsized":"2024-09-21T10:34:44.937245+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:47:17.004820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":200,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938094+0000","last_change":"2024-09-21T10:33:37.001489+0000","last_active":"2024-09-21T10:34:44.938094+0000","last_peered":"2024-09-21T10:34:44.938094+0000","last_clean":"2024-09-21T10:34:44.938094+0000","last_became_active":"2024-09-21T10:33:37.001116+0000","last_became_peered":"2024-09-21T10:33:37.001116+0000","last_unstale":"2024-09-21T10:34:44.938094+0000","last_undegraded":"2024-09-21T10:34:44.938094+0000","last_fullsized":"2024-09-21T10:34:44.938094+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:05:29.964124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":844,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858035+0000","last_change":"2024-09-21T10:33:39.992102+0000","last_active":"2024-09-21T10:34:44.858035+0000","last_peered":"2024-09-21T10:34:44.858035+0000","last_clean":"2024-09-21T10:34:44.858035+0000","last_became_active":"2024-09-21T10:33:39.991701+0000","last_became_peered":"2024-09-21T10:33:39.991701+0000","last_unstale":"2024-09-21T10:34:44.858035+0000","last_undegraded":"2024-09-21T10:34:44.858035+0000","last_fullsized":"2024-09-21T10:34:44.858035+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:22.717550+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:30:22.717550+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:45.929549+0000","scrub_duration":0.0073004899999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1362,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858151+0000","last_change":"2024-09-21T10:30:51.982312+0000","last_active":"2024-09-21T10:34:44.858151+0000","last_peered":"2024-09-21T10:34:44.858151+0000","last_clean":"2024-09-21T10:34:44.858151+0000","last_became_active":"2024-09-21T10:30:51.981405+0000","last_became_peered":"2024-09-21T10:30:51.981405+0000","last_unstale":"2024-09-21T10:34:44.858151+0000","last_undegraded":"2024-09-21T10:34:44.858151+0000","last_fullsized":"2024-09-21T10:34:44.858151+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:15.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":268,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226127+0000","last_change":"2024-09-21T10:33:36.995672+0000","last_active":"2024-09-21T10:34:41.226127+0000","last_peered":"2024-09-21T10:34:41.226127+0000","last_clean":"2024-09-21T10:34:41.226127+0000","last_became_active":"2024-09-21T10:33:36.995193+0000","last_became_peered":"2024-09-21T10:33:36.995193+0000","last_unstale":"2024-09-21T10:34:41.226127+0000","last_undegraded":"2024-09-21T10:34:41.226127+0000","last_fullsized":"2024-09-21T10:34:41.226127+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:24.513124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938003+0000","last_change":"2024-09-21T10:30:45.210421+0000","last_active":"2024-09-21T10:34:44.938003+0000","last_peered":"2024-09-21T10:34:44.938003+0000","last_clean":"2024-09-21T10:34:44.938003+0000","last_became_active":"2024-09-21T10:30:45.206024+0000","last_became_peered":"2024-09-21T10:30:45.206024+0000","last_unstale":"2024-09-21T10:34:44.938003+0000","last_undegraded":"2024-09-21T10:34:44.938003+0000","last_fullsized":"2024-09-21T10:34:44.938003+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:51.844704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":671,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938180+0000","last_change":"2024-09-21T10:34:29.176018+0000","last_active":"2024-09-21T10:34:44.938180+0000","last_peered":"2024-09-21T10:34:44.938180+0000","last_clean":"2024-09-21T10:34:44.938180+0000","last_became_active":"2024-09-21T10:34:29.175414+0000","last_became_peered":"2024-09-21T10:34:29.175414+0000","last_unstale":"2024-09-21T10:34:44.938180+0000","last_undegraded":"2024-09-21T10:34:44.938180+0000","last_fullsized":"2024-09-21T10:34:44.938180+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:31:00.915274+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:31:00.915274+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:16.839294+0000","scrub_duration":0.0051508819999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":481,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857531+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:34:44.857531+0000","last_peered":"2024-09-21T10:34:44.857531+0000","last_clean":"2024-09-21T10:34:44.857531+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:34:44.857531+0000","last_undegraded":"2024-09-21T10:34:44.857531+0000","last_fullsized":"2024-09-21T10:34:44.857531+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":983,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937770+0000","last_change":"2024-09-21T10:33:40.002608+0000","last_active":"2024-09-21T10:34:44.937770+0000","last_peered":"2024-09-21T10:34:44.937770+0000","last_clean":"2024-09-21T10:34:44.937770+0000","last_became_active":"2024-09-21T10:33:40.001927+0000","last_became_peered":"2024-09-21T10:33:40.001927+0000","last_unstale":"2024-09-21T10:34:44.937770+0000","last_undegraded":"2024-09-21T10:34:44.937770+0000","last_fullsized":"2024-09-21T10:34:44.937770+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:39:53.531362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937439+0000","last_change":"2024-09-21T10:30:45.223025+0000","last_active":"2024-09-21T10:34:44.937439+0000","last_peered":"2024-09-21T10:34:44.937439+0000","last_clean":"2024-09-21T10:34:44.937439+0000","last_became_active":"2024-09-21T10:30:45.222585+0000","last_became_peered":"2024-09-21T10:30:45.222585+0000","last_unstale":"2024-09-21T10:34:44.937439+0000","last_undegraded":"2024-09-21T10:34:44.937439+0000","last_fullsized":"2024-09-21T10:34:44.937439+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:14.096008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":692,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938559+0000","last_change":"2024-09-21T10:33:39.998237+0000","last_active":"2024-09-21T10:34:44.938559+0000","last_peered":"2024-09-21T10:34:44.938559+0000","last_clean":"2024-09-21T10:34:44.938559+0000","last_became_active":"2024-09-21T10:33:39.997793+0000","last_became_peered":"2024-09-21T10:33:39.997793+0000","last_unstale":"2024-09-21T10:34:44.938559+0000","last_undegraded":"2024-09-21T10:34:44.938559+0000","last_fullsized":"2024-09-21T10:34:44.938559+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:26.906418+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:26.906418+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:36.945992+0000","scrub_duration":0.0087083520000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937905+0000","last_change":"2024-09-21T10:30:45.210705+0000","last_active":"2024-09-21T10:34:44.937905+0000","last_peered":"2024-09-21T10:34:44.937905+0000","last_clean":"2024-09-21T10:34:44.937905+0000","last_became_active":"2024-09-21T10:30:45.205528+0000","last_became_peered":"2024-09-21T10:30:45.205528+0000","last_unstale":"2024-09-21T10:34:44.937905+0000","last_undegraded":"2024-09-21T10:34:44.937905+0000","last_fullsized":"2024-09-21T10:34:44.937905+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:32.515942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":1174,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938223+0000","last_change":"2024-09-21T10:33:37.023942+0000","last_active":"2024-09-21T10:34:44.938223+0000","last_peered":"2024-09-21T10:34:44.938223+0000","last_clean":"2024-09-21T10:34:44.938223+0000","last_became_active":"2024-09-21T10:33:37.001270+0000","last_became_peered":"2024-09-21T10:33:37.001270+0000","last_unstale":"2024-09-21T10:34:44.938223+0000","last_undegraded":"2024-09-21T10:34:44.938223+0000","last_fullsized":"2024-09-21T10:34:44.938223+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:16:01.086134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937209+0000","last_change":"2024-09-21T10:33:36.999948+0000","last_active":"2024-09-21T10:34:44.937209+0000","last_peered":"2024-09-21T10:34:44.937209+0000","last_clean":"2024-09-21T10:34:44.937209+0000","last_became_active":"2024-09-21T10:33:36.999416+0000","last_became_peered":"2024-09-21T10:33:36.999416+0000","last_unstale":"2024-09-21T10:34:44.937209+0000","last_undegraded":"2024-09-21T10:34:44.937209+0000","last_fullsized":"2024-09-21T10:34:44.937209+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:15:33.790607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":789,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858203+0000","last_change":"2024-09-21T10:30:22.584185+0000","last_active":"2024-09-21T10:34:44.858203+0000","last_peered":"2024-09-21T10:34:44.858203+0000","last_clean":"2024-09-21T10:34:44.858203+0000","last_became_active":"2024-09-21T10:30:22.056768+0000","last_became_peered":"2024-09-21T10:30:22.056768+0000","last_unstale":"2024-09-21T10:34:44.858203+0000","last_undegraded":"2024-09-21T10:34:44.858203+0000","last_fullsized":"2024-09-21T10:34:44.858203+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:22.583825+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:22.583825+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:59.427627+0000","scrub_duration":0.0051540479999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1009,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857919+0000","last_change":"2024-09-21T10:33:37.012499+0000","last_active":"2024-09-21T10:34:44.857919+0000","last_peered":"2024-09-21T10:34:44.857919+0000","last_clean":"2024-09-21T10:34:44.857919+0000","last_became_active":"2024-09-21T10:33:36.998155+0000","last_became_peered":"2024-09-21T10:33:36.998155+0000","last_unstale":"2024-09-21T10:34:44.857919+0000","last_undegraded":"2024-09-21T10:34:44.857919+0000","last_fullsized":"2024-09-21T10:34:44.857919+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:46.183541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":841,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938238+0000","last_change":"2024-09-21T10:30:51.986197+0000","last_active":"2024-09-21T10:34:44.938238+0000","last_peered":"2024-09-21T10:34:44.938238+0000","last_clean":"2024-09-21T10:34:44.938238+0000","last_became_active":"2024-09-21T10:30:51.985466+0000","last_became_peered":"2024-09-21T10:30:51.985466+0000","last_unstale":"2024-09-21T10:34:44.938238+0000","last_undegraded":"2024-09-21T10:34:44.938238+0000","last_fullsized":"2024-09-21T10:34:44.938238+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:16.999338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1011,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857046+0000","last_change":"2024-09-21T10:34:26.158778+0000","last_active":"2024-09-21T10:34:44.857046+0000","last_peered":"2024-09-21T10:34:44.857046+0000","last_clean":"2024-09-21T10:34:44.857046+0000","last_became_active":"2024-09-21T10:34:26.158428+0000","last_became_peered":"2024-09-21T10:34:26.158428+0000","last_unstale":"2024-09-21T10:34:44.857046+0000","last_undegraded":"2024-09-21T10:34:44.857046+0000","last_fullsized":"2024-09-21T10:34:44.857046+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:08:54.150856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":970,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857542+0000","last_change":"2024-09-21T10:30:22.757965+0000","last_active":"2024-09-21T10:34:44.857542+0000","last_peered":"2024-09-21T10:34:44.857542+0000","last_clean":"2024-09-21T10:34:44.857542+0000","last_became_active":"2024-09-21T10:30:22.041873+0000","last_became_peered":"2024-09-21T10:30:22.041873+0000","last_unstale":"2024-09-21T10:34:44.857542+0000","last_undegraded":"2024-09-21T10:34:44.857542+0000","last_fullsized":"2024-09-21T10:34:44.857542+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:30:22.757772+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:30:22.757772+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:17.470049+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":66,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937510+0000","last_change":"2024-09-21T10:33:37.013764+0000","last_active":"2024-09-21T10:34:44.937510+0000","last_peered":"2024-09-21T10:34:44.937510+0000","last_clean":"2024-09-21T10:34:44.937510+0000","last_became_active":"2024-09-21T10:33:37.001151+0000","last_became_peered":"2024-09-21T10:33:37.001151+0000","last_unstale":"2024-09-21T10:34:44.937510+0000","last_undegraded":"2024-09-21T10:34:44.937510+0000","last_fullsized":"2024-09-21T10:34:44.937510+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:39:51.809582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":905,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.694915+0000","last_change":"2024-09-21T10:34:26.170165+0000","last_active":"2024-09-21T10:34:44.694915+0000","last_peered":"2024-09-21T10:34:44.694915+0000","last_clean":"2024-09-21T10:34:44.694915+0000","last_became_active":"2024-09-21T10:34:26.156140+0000","last_became_peered":"2024-09-21T10:34:26.156140+0000","last_unstale":"2024-09-21T10:34:44.694915+0000","last_undegraded":"2024-09-21T10:34:44.694915+0000","last_fullsized":"2024-09-21T10:34:44.694915+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:06.763124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":505,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226555+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:34:41.226555+0000","last_peered":"2024-09-21T10:34:41.226555+0000","last_clean":"2024-09-21T10:34:41.226555+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:34:41.226555+0000","last_undegraded":"2024-09-21T10:34:41.226555+0000","last_fullsized":"2024-09-21T10:34:41.226555+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":547,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857813+0000","last_change":"2024-09-21T10:30:45.220621+0000","last_active":"2024-09-21T10:34:44.857813+0000","last_peered":"2024-09-21T10:34:44.857813+0000","last_clean":"2024-09-21T10:34:44.857813+0000","last_became_active":"2024-09-21T10:30:45.219593+0000","last_became_peered":"2024-09-21T10:30:45.219593+0000","last_unstale":"2024-09-21T10:34:44.857813+0000","last_undegraded":"2024-09-21T10:34:44.857813+0000","last_fullsized":"2024-09-21T10:34:44.857813+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:27:44.914386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":238,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226696+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:34:41.226696+0000","last_peered":"2024-09-21T10:34:41.226696+0000","last_clean":"2024-09-21T10:34:41.226696+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:34:41.226696+0000","last_undegraded":"2024-09-21T10:34:41.226696+0000","last_fullsized":"2024-09-21T10:34:41.226696+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937590+0000","last_change":"2024-09-21T10:30:45.217613+0000","last_active":"2024-09-21T10:34:44.937590+0000","last_peered":"2024-09-21T10:34:44.937590+0000","last_clean":"2024-09-21T10:34:44.937590+0000","last_became_active":"2024-09-21T10:30:45.217188+0000","last_became_peered":"2024-09-21T10:30:45.217188+0000","last_unstale":"2024-09-21T10:34:44.937590+0000","last_undegraded":"2024-09-21T10:34:44.937590+0000","last_fullsized":"2024-09-21T10:34:44.937590+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:54.177397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":24,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.478599+0000","last_change":"2024-09-21T10:34:26.164361+0000","last_active":"2024-09-21T10:34:40.478599+0000","last_peered":"2024-09-21T10:34:40.478599+0000","last_clean":"2024-09-21T10:34:40.478599+0000","last_became_active":"2024-09-21T10:34:26.159487+0000","last_became_peered":"2024-09-21T10:34:26.159487+0000","last_unstale":"2024-09-21T10:34:40.478599+0000","last_undegraded":"2024-09-21T10:34:40.478599+0000","last_fullsized":"2024-09-21T10:34:40.478599+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:34:19.301610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"373'150","reported_seq":973,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937785+0000","last_change":"2024-09-21T10:30:31.115487+0000","last_active":"2024-09-21T10:34:44.937785+0000","last_peered":"2024-09-21T10:34:44.937785+0000","last_clean":"2024-09-21T10:34:44.937785+0000","last_became_active":"2024-09-21T10:30:31.114941+0000","last_became_peered":"2024-09-21T10:30:31.114941+0000","last_unstale":"2024-09-21T10:34:44.937785+0000","last_undegraded":"2024-09-21T10:34:44.937785+0000","last_fullsized":"2024-09-21T10:34:44.937785+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:30:23.771955+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:30:23.771955+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:20:42.604348+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":258,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.353704+0000","last_change":"2024-09-21T10:30:45.199098+0000","last_active":"2024-09-21T10:34:41.353704+0000","last_peered":"2024-09-21T10:34:41.353704+0000","last_clean":"2024-09-21T10:34:41.353704+0000","last_became_active":"2024-09-21T10:30:45.198762+0000","last_became_peered":"2024-09-21T10:30:45.198762+0000","last_unstale":"2024-09-21T10:34:41.353704+0000","last_undegraded":"2024-09-21T10:34:41.353704+0000","last_fullsized":"2024-09-21T10:34:41.353704+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:40.566015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":905,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857123+0000","last_change":"2024-09-21T10:34:26.158771+0000","last_active":"2024-09-21T10:34:44.857123+0000","last_peered":"2024-09-21T10:34:44.857123+0000","last_clean":"2024-09-21T10:34:44.857123+0000","last_became_active":"2024-09-21T10:34:26.158381+0000","last_became_peered":"2024-09-21T10:34:26.158381+0000","last_unstale":"2024-09-21T10:34:44.857123+0000","last_undegraded":"2024-09-21T10:34:44.857123+0000","last_fullsized":"2024-09-21T10:34:44.857123+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:23.238104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937565+0000","last_change":"2024-09-21T10:33:37.005046+0000","last_active":"2024-09-21T10:34:44.937565+0000","last_peered":"2024-09-21T10:34:44.937565+0000","last_clean":"2024-09-21T10:34:44.937565+0000","last_became_active":"2024-09-21T10:33:37.004704+0000","last_became_peered":"2024-09-21T10:33:37.004704+0000","last_unstale":"2024-09-21T10:34:44.937565+0000","last_undegraded":"2024-09-21T10:34:44.937565+0000","last_fullsized":"2024-09-21T10:34:44.937565+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:36.930972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":972,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.856996+0000","last_change":"2024-09-21T10:33:22.277354+0000","last_active":"2024-09-21T10:34:44.856996+0000","last_peered":"2024-09-21T10:34:44.856996+0000","last_clean":"2024-09-21T10:34:44.856996+0000","last_became_active":"2024-09-21T10:33:22.276984+0000","last_became_peered":"2024-09-21T10:33:22.276984+0000","last_unstale":"2024-09-21T10:34:44.856996+0000","last_undegraded":"2024-09-21T10:34:44.856996+0000","last_fullsized":"2024-09-21T10:34:44.856996+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:43.527258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":565,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906776+0000","last_change":"2024-09-21T10:30:45.209630+0000","last_active":"2024-09-21T10:34:40.906776+0000","last_peered":"2024-09-21T10:34:40.906776+0000","last_clean":"2024-09-21T10:34:40.906776+0000","last_became_active":"2024-09-21T10:30:45.209304+0000","last_became_peered":"2024-09-21T10:30:45.209304+0000","last_unstale":"2024-09-21T10:34:40.906776+0000","last_undegraded":"2024-09-21T10:34:40.906776+0000","last_fullsized":"2024-09-21T10:34:40.906776+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:29.693394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1011,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857118+0000","last_change":"2024-09-21T10:33:36.998197+0000","last_active":"2024-09-21T10:34:44.857118+0000","last_peered":"2024-09-21T10:34:44.857118+0000","last_clean":"2024-09-21T10:34:44.857118+0000","last_became_active":"2024-09-21T10:33:36.997830+0000","last_became_peered":"2024-09-21T10:33:36.997830+0000","last_unstale":"2024-09-21T10:34:44.857118+0000","last_undegraded":"2024-09-21T10:34:44.857118+0000","last_fullsized":"2024-09-21T10:34:44.857118+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:27.718538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":69,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937398+0000","last_change":"2024-09-21T10:34:26.161955+0000","last_active":"2024-09-21T10:34:44.937398+0000","last_peered":"2024-09-21T10:34:44.937398+0000","last_clean":"2024-09-21T10:34:44.937398+0000","last_became_active":"2024-09-21T10:34:26.161618+0000","last_became_peered":"2024-09-21T10:34:26.161618+0000","last_unstale":"2024-09-21T10:34:44.937398+0000","last_undegraded":"2024-09-21T10:34:44.937398+0000","last_fullsized":"2024-09-21T10:34:44.937398+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:53:04.405320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":181,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937729+0000","last_change":"2024-09-21T10:30:45.206647+0000","last_active":"2024-09-21T10:34:44.937729+0000","last_peered":"2024-09-21T10:34:44.937729+0000","last_clean":"2024-09-21T10:34:44.937729+0000","last_became_active":"2024-09-21T10:30:45.205723+0000","last_became_peered":"2024-09-21T10:30:45.205723+0000","last_unstale":"2024-09-21T10:34:44.937729+0000","last_undegraded":"2024-09-21T10:34:44.937729+0000","last_fullsized":"2024-09-21T10:34:44.937729+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:55:33.281876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":971,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857541+0000","last_change":"2024-09-21T10:30:45.207518+0000","last_active":"2024-09-21T10:34:44.857541+0000","last_peered":"2024-09-21T10:34:44.857541+0000","last_clean":"2024-09-21T10:34:44.857541+0000","last_became_active":"2024-09-21T10:30:45.207075+0000","last_became_peered":"2024-09-21T10:30:45.207075+0000","last_unstale":"2024-09-21T10:34:44.857541+0000","last_undegraded":"2024-09-21T10:34:44.857541+0000","last_fullsized":"2024-09-21T10:34:44.857541+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:33.239764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":554,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857749+0000","last_change":"2024-09-21T10:30:45.222126+0000","last_active":"2024-09-21T10:34:44.857749+0000","last_peered":"2024-09-21T10:34:44.857749+0000","last_clean":"2024-09-21T10:34:44.857749+0000","last_became_active":"2024-09-21T10:30:45.221747+0000","last_became_peered":"2024-09-21T10:30:45.221747+0000","last_unstale":"2024-09-21T10:34:44.857749+0000","last_undegraded":"2024-09-21T10:34:44.857749+0000","last_fullsized":"2024-09-21T10:34:44.857749+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:15.856857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":970,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937894+0000","last_change":"2024-09-21T10:30:45.216426+0000","last_active":"2024-09-21T10:34:44.937894+0000","last_peered":"2024-09-21T10:34:44.937894+0000","last_clean":"2024-09-21T10:34:44.937894+0000","last_became_active":"2024-09-21T10:30:45.215768+0000","last_became_peered":"2024-09-21T10:30:45.215768+0000","last_unstale":"2024-09-21T10:34:44.937894+0000","last_undegraded":"2024-09-21T10:34:44.937894+0000","last_fullsized":"2024-09-21T10:34:44.937894+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:13.389909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937097+0000","last_change":"2024-09-21T10:33:37.001740+0000","last_active":"2024-09-21T10:34:44.937097+0000","last_peered":"2024-09-21T10:34:44.937097+0000","last_clean":"2024-09-21T10:34:44.937097+0000","last_became_active":"2024-09-21T10:33:37.001289+0000","last_became_peered":"2024-09-21T10:33:37.001289+0000","last_unstale":"2024-09-21T10:34:44.937097+0000","last_undegraded":"2024-09-21T10:34:44.937097+0000","last_fullsized":"2024-09-21T10:34:44.937097+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:25:13.491213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":255,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857588+0000","last_change":"2024-09-21T10:34:22.335483+0000","last_active":"2024-09-21T10:34:44.857588+0000","last_peered":"2024-09-21T10:34:44.857588+0000","last_clean":"2024-09-21T10:34:44.857588+0000","last_became_active":"2024-09-21T10:34:22.335082+0000","last_became_peered":"2024-09-21T10:34:22.335082+0000","last_unstale":"2024-09-21T10:34:44.857588+0000","last_undegraded":"2024-09-21T10:34:44.857588+0000","last_fullsized":"2024-09-21T10:34:44.857588+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:10:48.483607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":498,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858342+0000","last_change":"2024-09-21T10:30:45.220768+0000","last_active":"2024-09-21T10:34:44.858342+0000","last_peered":"2024-09-21T10:34:44.858342+0000","last_clean":"2024-09-21T10:34:44.858342+0000","last_became_active":"2024-09-21T10:30:45.214059+0000","last_became_peered":"2024-09-21T10:30:45.214059+0000","last_unstale":"2024-09-21T10:34:44.858342+0000","last_undegraded":"2024-09-21T10:34:44.858342+0000","last_fullsized":"2024-09-21T10:34:44.858342+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:12:52.858780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":267,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857250+0000","last_change":"2024-09-21T10:33:36.995401+0000","last_active":"2024-09-21T10:34:44.857250+0000","last_peered":"2024-09-21T10:34:44.857250+0000","last_clean":"2024-09-21T10:34:44.857250+0000","last_became_active":"2024-09-21T10:33:36.995022+0000","last_became_peered":"2024-09-21T10:33:36.995022+0000","last_unstale":"2024-09-21T10:34:44.857250+0000","last_undegraded":"2024-09-21T10:34:44.857250+0000","last_fullsized":"2024-09-21T10:34:44.857250+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:15.343376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":450,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858602+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:34:44.858602+0000","last_peered":"2024-09-21T10:34:44.858602+0000","last_clean":"2024-09-21T10:34:44.858602+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:34:44.858602+0000","last_undegraded":"2024-09-21T10:34:44.858602+0000","last_fullsized":"2024-09-21T10:34:44.858602+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1000,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857467+0000","last_change":"2024-09-21T10:33:36.996843+0000","last_active":"2024-09-21T10:34:44.857467+0000","last_peered":"2024-09-21T10:34:44.857467+0000","last_clean":"2024-09-21T10:34:44.857467+0000","last_became_active":"2024-09-21T10:33:36.996479+0000","last_became_peered":"2024-09-21T10:33:36.996479+0000","last_unstale":"2024-09-21T10:34:44.857467+0000","last_undegraded":"2024-09-21T10:34:44.857467+0000","last_fullsized":"2024-09-21T10:34:44.857467+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:09:56.739587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":933,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857762+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:34:44.857762+0000","last_peered":"2024-09-21T10:34:44.857762+0000","last_clean":"2024-09-21T10:34:44.857762+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:34:44.857762+0000","last_undegraded":"2024-09-21T10:34:44.857762+0000","last_fullsized":"2024-09-21T10:34:44.857762+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":246,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226211+0000","last_change":"2024-09-21T10:30:45.206962+0000","last_active":"2024-09-21T10:34:41.226211+0000","last_peered":"2024-09-21T10:34:41.226211+0000","last_clean":"2024-09-21T10:34:41.226211+0000","last_became_active":"2024-09-21T10:30:45.206174+0000","last_became_peered":"2024-09-21T10:30:45.206174+0000","last_unstale":"2024-09-21T10:34:41.226211+0000","last_undegraded":"2024-09-21T10:34:41.226211+0000","last_fullsized":"2024-09-21T10:34:41.226211+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:03:00.246788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":68,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937446+0000","last_change":"2024-09-21T10:33:37.011028+0000","last_active":"2024-09-21T10:34:44.937446+0000","last_peered":"2024-09-21T10:34:44.937446+0000","last_clean":"2024-09-21T10:34:44.937446+0000","last_became_active":"2024-09-21T10:33:36.998998+0000","last_became_peered":"2024-09-21T10:33:36.998998+0000","last_unstale":"2024-09-21T10:34:44.937446+0000","last_undegraded":"2024-09-21T10:34:44.937446+0000","last_fullsized":"2024-09-21T10:34:44.937446+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:01.037536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":6644,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30003,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":376,"num_bytes_recovered":33554489,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3133,"ondisk_log_size":3133,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234551,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":958188,"kb_used_data":10720,"kb_used_omap":1,"kb_used_meta":947454,"kb_avail":92786964,"statfs":{"total":95995035648,"available":95013851136,"internally_reserved":0,"allocated":10977280,"data_stored":8366461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":970193355},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496189,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904360,"kb_used_data":6940,"kb_used_omap":1,"kb_used_meta":897406,"kb_avail":92840792,"statfs":{"total":95995035648,"available":95068971008,"internally_reserved":0,"allocated":7106560,"data_stored":4500041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":918944199},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569404088,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":658212,"kb_used_data":6872,"kb_used_omap":7,"kb_used_meta":651320,"kb_avail":93086940,"statfs":{"total":95995035648,"available":95321026560,"internally_reserved":0,"allocated":7036928,"data_stored":4430875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":666952600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019462,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":876140,"kb_used_data":6624,"kb_used_omap":1,"kb_used_meta":869502,"kb_avail":92869012,"statfs":{"total":95995035648,"available":95097868288,"internally_reserved":0,"allocated":6782976,"data_stored":4172418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":890370502},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435646,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1159644,"kb_used_data":7184,"kb_used_omap":5,"kb_used_meta":1152442,"kb_avail":92585508,"statfs":{"total":95995035648,"available":94807560192,"internally_reserved":0,"allocated":7356416,"data_stored":4757986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":1180100648},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599173,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":949276,"kb_used_data":6864,"kb_used_omap":7,"kb_used_meta":942392,"kb_avail":92795876,"statfs":{"total":95995035648,"available":95022977024,"internally_reserved":0,"allocated":7028736,"data_stored":4430524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":965009869},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762700,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":940628,"kb_used_data":10696,"kb_used_omap":7,"kb_used_meta":929912,"kb_avail":92804524,"statfs":{"total":95995035648,"available":95031832576,"internally_reserved":0,"allocated":10952704,"data_stored":8361578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":952230141},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":491,"seq":2108828942365,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":78364,"kb_used_data":11024,"kb_used_omap":16,"kb_used_meta":67311,"kb_avail":93666788,"statfs":{"total":95995035648,"available":95914790912,"internally_reserved":0,"allocated":11288576,"data_stored":8690116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16669,"internal_metadata":68927203},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12186,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:35:33.062 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:35:33.273 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:35:33.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:33 smithi045 ceph-mon[95976]: from='client.46664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:35:33.381 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:33 smithi032 ceph-mon[105748]: from='client.46664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:35:33.381 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:33 smithi032 ceph-mon[101299]: from='client.46664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:35:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:34 smithi045 ceph-mon[95976]: pgmap v1618: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:34.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:34 smithi045 ceph-mon[95976]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:35:34.426 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:34 smithi032 ceph-mon[105748]: pgmap v1618: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:34.427 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:34 smithi032 ceph-mon[105748]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:35:34.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:34 smithi032 ceph-mon[101299]: pgmap v1618: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:34.427 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:34 smithi032 ceph-mon[101299]: from='mgr.25736 172.21.15.32:0/631656290' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:35:35.139 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-09-21T10:35:35.144 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-09-21T10:35:35.793 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":1619,"stamp":"2024-09-21T10:35:34.812159+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":908,"num_write_kb":12625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":461,"num_bytes_recovered":34544173,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4166,"ondisk_log_size":4166,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6524812,"kb_used_data":66924,"kb_used_omap":49,"kb_used_meta":6457742,"kb_avail":743436404,"statfs":{"total":767960285184,"available":761278877696,"internally_reserved":0,"allocated":68530176,"data_stored":47709999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":50491,"internal_metadata":6612728517},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003024"},"pg_stats":[{"pgid":"1.11","version":"90'149","reported_seq":779,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937319+0000","last_change":"2024-09-21T10:33:39.997985+0000","last_active":"2024-09-21T10:34:44.937319+0000","last_peered":"2024-09-21T10:34:44.937319+0000","last_clean":"2024-09-21T10:34:44.937319+0000","last_became_active":"2024-09-21T10:33:39.997631+0000","last_became_peered":"2024-09-21T10:33:39.997631+0000","last_unstale":"2024-09-21T10:34:44.937319+0000","last_undegraded":"2024-09-21T10:34:44.937319+0000","last_fullsized":"2024-09-21T10:34:44.937319+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:26.643960+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:30:26.643960+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:24.384086+0000","scrub_duration":0.013444612,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":952,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857669+0000","last_change":"2024-09-21T10:30:45.210213+0000","last_active":"2024-09-21T10:34:44.857669+0000","last_peered":"2024-09-21T10:34:44.857669+0000","last_clean":"2024-09-21T10:34:44.857669+0000","last_became_active":"2024-09-21T10:30:45.207807+0000","last_became_peered":"2024-09-21T10:30:45.207807+0000","last_unstale":"2024-09-21T10:34:44.857669+0000","last_undegraded":"2024-09-21T10:34:44.857669+0000","last_fullsized":"2024-09-21T10:34:44.857669+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:00.839446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":536,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.526423+0000","last_change":"2024-09-21T10:19:50.586108+0000","last_active":"2024-09-21T10:34:44.526423+0000","last_peered":"2024-09-21T10:34:44.526423+0000","last_clean":"2024-09-21T10:34:44.526423+0000","last_became_active":"2024-09-21T10:19:50.578656+0000","last_became_peered":"2024-09-21T10:19:50.578656+0000","last_unstale":"2024-09-21T10:34:44.526423+0000","last_undegraded":"2024-09-21T10:34:44.526423+0000","last_fullsized":"2024-09-21T10:34:44.526423+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:34.058871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'149","reported_seq":707,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857446+0000","last_change":"2024-09-21T10:30:58.557091+0000","last_active":"2024-09-21T10:34:44.857446+0000","last_peered":"2024-09-21T10:34:44.857446+0000","last_clean":"2024-09-21T10:34:44.857446+0000","last_became_active":"2024-09-21T10:30:31.102167+0000","last_became_peered":"2024-09-21T10:30:31.102167+0000","last_unstale":"2024-09-21T10:34:44.857446+0000","last_undegraded":"2024-09-21T10:34:44.857446+0000","last_fullsized":"2024-09-21T10:34:44.857446+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:58.556988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:58.556988+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:55.249909+0000","scrub_duration":0.0013450059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":232,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.353692+0000","last_change":"2024-09-21T10:20:13.081319+0000","last_active":"2024-09-21T10:34:41.353692+0000","last_peered":"2024-09-21T10:34:41.353692+0000","last_clean":"2024-09-21T10:34:41.353692+0000","last_became_active":"2024-09-21T10:20:13.081025+0000","last_became_peered":"2024-09-21T10:20:13.081025+0000","last_unstale":"2024-09-21T10:34:41.353692+0000","last_undegraded":"2024-09-21T10:34:41.353692+0000","last_fullsized":"2024-09-21T10:34:41.353692+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:19.885083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":69,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937024+0000","last_change":"2024-09-21T10:34:26.160198+0000","last_active":"2024-09-21T10:34:44.937024+0000","last_peered":"2024-09-21T10:34:44.937024+0000","last_clean":"2024-09-21T10:34:44.937024+0000","last_became_active":"2024-09-21T10:34:26.159818+0000","last_became_peered":"2024-09-21T10:34:26.159818+0000","last_unstale":"2024-09-21T10:34:44.937024+0000","last_undegraded":"2024-09-21T10:34:44.937024+0000","last_fullsized":"2024-09-21T10:34:44.937024+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:53.948099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"90'149","reported_seq":921,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937794+0000","last_change":"2024-09-21T10:30:47.984484+0000","last_active":"2024-09-21T10:34:44.937794+0000","last_peered":"2024-09-21T10:34:44.937794+0000","last_clean":"2024-09-21T10:34:44.937794+0000","last_became_active":"2024-09-21T10:30:47.981723+0000","last_became_peered":"2024-09-21T10:30:47.981723+0000","last_unstale":"2024-09-21T10:34:44.937794+0000","last_undegraded":"2024-09-21T10:34:44.937794+0000","last_fullsized":"2024-09-21T10:34:44.937794+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:26.687488+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:30:26.687488+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:17.623163+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":69,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937516+0000","last_change":"2024-09-21T10:34:26.161153+0000","last_active":"2024-09-21T10:34:44.937516+0000","last_peered":"2024-09-21T10:34:44.937516+0000","last_clean":"2024-09-21T10:34:44.937516+0000","last_became_active":"2024-09-21T10:34:26.160837+0000","last_became_peered":"2024-09-21T10:34:26.160837+0000","last_unstale":"2024-09-21T10:34:44.937516+0000","last_undegraded":"2024-09-21T10:34:44.937516+0000","last_fullsized":"2024-09-21T10:34:44.937516+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:09.108139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":507,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.526389+0000","last_change":"2024-09-21T10:19:50.551906+0000","last_active":"2024-09-21T10:34:44.526389+0000","last_peered":"2024-09-21T10:34:44.526389+0000","last_clean":"2024-09-21T10:34:44.526389+0000","last_became_active":"2024-09-21T10:19:50.551501+0000","last_became_peered":"2024-09-21T10:19:50.551501+0000","last_unstale":"2024-09-21T10:34:44.526389+0000","last_undegraded":"2024-09-21T10:34:44.526389+0000","last_fullsized":"2024-09-21T10:34:44.526389+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:48.742092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'149","reported_seq":741,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857925+0000","last_change":"2024-09-21T10:30:59.599305+0000","last_active":"2024-09-21T10:34:44.857925+0000","last_peered":"2024-09-21T10:34:44.857925+0000","last_clean":"2024-09-21T10:34:44.857925+0000","last_became_active":"2024-09-21T10:30:47.960200+0000","last_became_peered":"2024-09-21T10:30:47.960200+0000","last_unstale":"2024-09-21T10:34:44.857925+0000","last_undegraded":"2024-09-21T10:34:44.857925+0000","last_fullsized":"2024-09-21T10:34:44.857925+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:59.599224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:59.599224+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:54.053533+0000","scrub_duration":0.001301359,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":987,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857878+0000","last_change":"2024-09-21T10:34:26.159055+0000","last_active":"2024-09-21T10:34:44.857878+0000","last_peered":"2024-09-21T10:34:44.857878+0000","last_clean":"2024-09-21T10:34:44.857878+0000","last_became_active":"2024-09-21T10:34:26.158706+0000","last_became_peered":"2024-09-21T10:34:26.158706+0000","last_unstale":"2024-09-21T10:34:44.857878+0000","last_undegraded":"2024-09-21T10:34:44.857878+0000","last_fullsized":"2024-09-21T10:34:44.857878+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:11.025996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":647,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906164+0000","last_change":"2024-09-21T10:33:37.006914+0000","last_active":"2024-09-21T10:34:40.906164+0000","last_peered":"2024-09-21T10:34:40.906164+0000","last_clean":"2024-09-21T10:34:40.906164+0000","last_became_active":"2024-09-21T10:33:36.994021+0000","last_became_peered":"2024-09-21T10:33:36.994021+0000","last_unstale":"2024-09-21T10:34:40.906164+0000","last_undegraded":"2024-09-21T10:34:40.906164+0000","last_fullsized":"2024-09-21T10:34:40.906164+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:25.986510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"90'149","reported_seq":661,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.478719+0000","last_change":"2024-09-21T10:34:32.934865+0000","last_active":"2024-09-21T10:34:40.478719+0000","last_peered":"2024-09-21T10:34:40.478719+0000","last_clean":"2024-09-21T10:34:40.478719+0000","last_became_active":"2024-09-21T10:34:29.178213+0000","last_became_peered":"2024-09-21T10:34:29.178213+0000","last_unstale":"2024-09-21T10:34:40.478719+0000","last_undegraded":"2024-09-21T10:34:40.478719+0000","last_fullsized":"2024-09-21T10:34:40.478719+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:34:32.934773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:34:32.934773+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:06.954855+0000","scrub_duration":0.0014781499999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":22,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.478805+0000","last_change":"2024-09-21T10:34:26.166495+0000","last_active":"2024-09-21T10:34:40.478805+0000","last_peered":"2024-09-21T10:34:40.478805+0000","last_clean":"2024-09-21T10:34:40.478805+0000","last_became_active":"2024-09-21T10:34:26.160915+0000","last_became_peered":"2024-09-21T10:34:26.160915+0000","last_unstale":"2024-09-21T10:34:40.478805+0000","last_undegraded":"2024-09-21T10:34:40.478805+0000","last_fullsized":"2024-09-21T10:34:40.478805+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:39:08.788925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":259,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857647+0000","last_change":"2024-09-21T10:34:26.154060+0000","last_active":"2024-09-21T10:34:44.857647+0000","last_peered":"2024-09-21T10:34:44.857647+0000","last_clean":"2024-09-21T10:34:44.857647+0000","last_became_active":"2024-09-21T10:34:26.153721+0000","last_became_peered":"2024-09-21T10:34:26.153721+0000","last_unstale":"2024-09-21T10:34:44.857647+0000","last_undegraded":"2024-09-21T10:34:44.857647+0000","last_fullsized":"2024-09-21T10:34:44.857647+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:21.988421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"373'150","reported_seq":1016,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857551+0000","last_change":"2024-09-21T10:33:39.996646+0000","last_active":"2024-09-21T10:34:44.857551+0000","last_peered":"2024-09-21T10:34:44.857551+0000","last_clean":"2024-09-21T10:34:44.857551+0000","last_became_active":"2024-09-21T10:33:39.996222+0000","last_became_peered":"2024-09-21T10:33:39.996222+0000","last_unstale":"2024-09-21T10:34:44.857551+0000","last_undegraded":"2024-09-21T10:34:44.857551+0000","last_fullsized":"2024-09-21T10:34:44.857551+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":4,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:30:24.812285+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:30:24.812285+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:40.967138+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":236,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938322+0000","last_change":"2024-09-21T10:30:45.225017+0000","last_active":"2024-09-21T10:34:44.938322+0000","last_peered":"2024-09-21T10:34:44.938322+0000","last_clean":"2024-09-21T10:34:44.938322+0000","last_became_active":"2024-09-21T10:30:45.214625+0000","last_became_peered":"2024-09-21T10:30:45.214625+0000","last_unstale":"2024-09-21T10:34:44.938322+0000","last_undegraded":"2024-09-21T10:34:44.938322+0000","last_fullsized":"2024-09-21T10:34:44.938322+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:41.994673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":247,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857170+0000","last_change":"2024-09-21T10:30:45.206472+0000","last_active":"2024-09-21T10:34:44.857170+0000","last_peered":"2024-09-21T10:34:44.857170+0000","last_clean":"2024-09-21T10:34:44.857170+0000","last_became_active":"2024-09-21T10:30:45.205955+0000","last_became_peered":"2024-09-21T10:30:45.205955+0000","last_unstale":"2024-09-21T10:34:44.857170+0000","last_undegraded":"2024-09-21T10:34:44.857170+0000","last_fullsized":"2024-09-21T10:34:44.857170+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:50.262089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'149","reported_seq":653,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938285+0000","last_change":"2024-09-21T10:31:01.909114+0000","last_active":"2024-09-21T10:34:44.938285+0000","last_peered":"2024-09-21T10:34:44.938285+0000","last_clean":"2024-09-21T10:34:44.938285+0000","last_became_active":"2024-09-21T10:30:31.109278+0000","last_became_peered":"2024-09-21T10:30:31.109278+0000","last_unstale":"2024-09-21T10:34:44.938285+0000","last_undegraded":"2024-09-21T10:34:44.938285+0000","last_fullsized":"2024-09-21T10:34:44.938285+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:31:01.905848+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:31:01.905848+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:16:19.880231+0000","scrub_duration":0.018313652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":532,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938126+0000","last_change":"2024-09-21T10:20:13.083252+0000","last_active":"2024-09-21T10:34:44.938126+0000","last_peered":"2024-09-21T10:34:44.938126+0000","last_clean":"2024-09-21T10:34:44.938126+0000","last_became_active":"2024-09-21T10:20:13.082911+0000","last_became_peered":"2024-09-21T10:20:13.082911+0000","last_unstale":"2024-09-21T10:34:44.938126+0000","last_undegraded":"2024-09-21T10:34:44.938126+0000","last_fullsized":"2024-09-21T10:34:44.938126+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:45.641578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":587,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906271+0000","last_change":"2024-09-21T10:34:26.157998+0000","last_active":"2024-09-21T10:34:40.906271+0000","last_peered":"2024-09-21T10:34:40.906271+0000","last_clean":"2024-09-21T10:34:40.906271+0000","last_became_active":"2024-09-21T10:34:26.157665+0000","last_became_peered":"2024-09-21T10:34:26.157665+0000","last_unstale":"2024-09-21T10:34:40.906271+0000","last_undegraded":"2024-09-21T10:34:40.906271+0000","last_fullsized":"2024-09-21T10:34:40.906271+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:47:06.546254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'149","reported_seq":707,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857736+0000","last_change":"2024-09-21T10:31:00.501007+0000","last_active":"2024-09-21T10:34:44.857736+0000","last_peered":"2024-09-21T10:34:44.857736+0000","last_clean":"2024-09-21T10:34:44.857736+0000","last_became_active":"2024-09-21T10:30:31.105952+0000","last_became_peered":"2024-09-21T10:30:31.105952+0000","last_unstale":"2024-09-21T10:34:44.857736+0000","last_undegraded":"2024-09-21T10:34:44.857736+0000","last_fullsized":"2024-09-21T10:34:44.857736+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:31:00.500754+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:31:00.500754+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:00.821304+0000","scrub_duration":0.0048757180000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":465,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.478163+0000","last_change":"2024-09-21T10:20:13.082122+0000","last_active":"2024-09-21T10:34:40.478163+0000","last_peered":"2024-09-21T10:34:40.478163+0000","last_clean":"2024-09-21T10:34:40.478163+0000","last_became_active":"2024-09-21T10:20:13.081850+0000","last_became_peered":"2024-09-21T10:20:13.081850+0000","last_unstale":"2024-09-21T10:34:40.478163+0000","last_undegraded":"2024-09-21T10:34:40.478163+0000","last_fullsized":"2024-09-21T10:34:40.478163+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:22.175472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937313+0000","last_change":"2024-09-21T10:33:37.001969+0000","last_active":"2024-09-21T10:34:44.937313+0000","last_peered":"2024-09-21T10:34:44.937313+0000","last_clean":"2024-09-21T10:34:44.937313+0000","last_became_active":"2024-09-21T10:33:37.000831+0000","last_became_peered":"2024-09-21T10:33:37.000831+0000","last_unstale":"2024-09-21T10:34:44.937313+0000","last_undegraded":"2024-09-21T10:34:44.937313+0000","last_fullsized":"2024-09-21T10:34:44.937313+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:17.652214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":817,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.477685+0000","last_change":"2024-09-21T10:24:24.830614+0000","last_active":"2024-09-21T10:34:40.477685+0000","last_peered":"2024-09-21T10:34:40.477685+0000","last_clean":"2024-09-21T10:34:40.477685+0000","last_became_active":"2024-09-21T10:24:24.830208+0000","last_became_peered":"2024-09-21T10:24:24.830208+0000","last_unstale":"2024-09-21T10:34:40.477685+0000","last_undegraded":"2024-09-21T10:34:40.477685+0000","last_fullsized":"2024-09-21T10:34:40.477685+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:24:11.163904+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:18:21.604320+0000","last_clean_scrub_stamp":"2024-09-21T10:24:11.163904+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:45.011933+0000","scrub_duration":0.0017196850000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1026,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857607+0000","last_change":"2024-09-21T10:33:37.018129+0000","last_active":"2024-09-21T10:34:44.857607+0000","last_peered":"2024-09-21T10:34:44.857607+0000","last_clean":"2024-09-21T10:34:44.857607+0000","last_became_active":"2024-09-21T10:33:36.999471+0000","last_became_peered":"2024-09-21T10:33:36.999471+0000","last_unstale":"2024-09-21T10:34:44.857607+0000","last_undegraded":"2024-09-21T10:34:44.857607+0000","last_fullsized":"2024-09-21T10:34:44.857607+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:01.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":229,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857416+0000","last_change":"2024-09-21T10:19:50.584964+0000","last_active":"2024-09-21T10:34:44.857416+0000","last_peered":"2024-09-21T10:34:44.857416+0000","last_clean":"2024-09-21T10:34:44.857416+0000","last_became_active":"2024-09-21T10:19:50.584629+0000","last_became_peered":"2024-09-21T10:19:50.584629+0000","last_unstale":"2024-09-21T10:34:44.857416+0000","last_undegraded":"2024-09-21T10:34:44.857416+0000","last_fullsized":"2024-09-21T10:34:44.857416+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:25:16.138733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"304'150","reported_seq":590,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857556+0000","last_change":"2024-09-21T10:25:09.671244+0000","last_active":"2024-09-21T10:34:44.857556+0000","last_peered":"2024-09-21T10:34:44.857556+0000","last_clean":"2024-09-21T10:34:44.857556+0000","last_became_active":"2024-09-21T10:24:24.829435+0000","last_became_peered":"2024-09-21T10:24:24.829435+0000","last_unstale":"2024-09-21T10:34:44.857556+0000","last_undegraded":"2024-09-21T10:34:44.857556+0000","last_fullsized":"2024-09-21T10:34:44.857556+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"304'150","last_scrub_stamp":"2024-09-21T10:25:09.671118+0000","last_deep_scrub":"304'150","last_deep_scrub_stamp":"2024-09-21T10:24:21.271541+0000","last_clean_scrub_stamp":"2024-09-21T10:25:09.671118+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:24.496226+0000","scrub_duration":0.0051635919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":559,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906406+0000","last_change":"2024-09-21T10:30:45.223044+0000","last_active":"2024-09-21T10:34:40.906406+0000","last_peered":"2024-09-21T10:34:40.906406+0000","last_clean":"2024-09-21T10:34:40.906406+0000","last_became_active":"2024-09-21T10:30:45.222166+0000","last_became_peered":"2024-09-21T10:30:45.222166+0000","last_unstale":"2024-09-21T10:34:40.906406+0000","last_undegraded":"2024-09-21T10:34:40.906406+0000","last_fullsized":"2024-09-21T10:34:40.906406+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:16.215392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":247,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858148+0000","last_change":"2024-09-21T10:30:45.206465+0000","last_active":"2024-09-21T10:34:44.858148+0000","last_peered":"2024-09-21T10:34:44.858148+0000","last_clean":"2024-09-21T10:34:44.858148+0000","last_became_active":"2024-09-21T10:30:45.206088+0000","last_became_peered":"2024-09-21T10:30:45.206088+0000","last_unstale":"2024-09-21T10:34:44.858148+0000","last_undegraded":"2024-09-21T10:34:44.858148+0000","last_fullsized":"2024-09-21T10:34:44.858148+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:46:01.557325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'149","reported_seq":965,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938096+0000","last_change":"2024-09-21T10:33:40.003862+0000","last_active":"2024-09-21T10:34:44.938096+0000","last_peered":"2024-09-21T10:34:44.938096+0000","last_clean":"2024-09-21T10:34:44.938096+0000","last_became_active":"2024-09-21T10:33:40.003416+0000","last_became_peered":"2024-09-21T10:33:40.003416+0000","last_unstale":"2024-09-21T10:34:44.938096+0000","last_undegraded":"2024-09-21T10:34:44.938096+0000","last_fullsized":"2024-09-21T10:34:44.938096+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:24.704596+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:30:24.704596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:56:28.377638+0000","scrub_duration":14,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":229,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857041+0000","last_change":"2024-09-21T10:19:50.550957+0000","last_active":"2024-09-21T10:34:44.857041+0000","last_peered":"2024-09-21T10:34:44.857041+0000","last_clean":"2024-09-21T10:34:44.857041+0000","last_became_active":"2024-09-21T10:19:50.550116+0000","last_became_peered":"2024-09-21T10:19:50.550116+0000","last_unstale":"2024-09-21T10:34:44.857041+0000","last_undegraded":"2024-09-21T10:34:44.857041+0000","last_fullsized":"2024-09-21T10:34:44.857041+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:44.700247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":246,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226998+0000","last_change":"2024-09-21T10:30:45.220586+0000","last_active":"2024-09-21T10:34:41.226998+0000","last_peered":"2024-09-21T10:34:41.226998+0000","last_clean":"2024-09-21T10:34:41.226998+0000","last_became_active":"2024-09-21T10:30:45.219727+0000","last_became_peered":"2024-09-21T10:30:45.219727+0000","last_unstale":"2024-09-21T10:34:41.226998+0000","last_undegraded":"2024-09-21T10:34:41.226998+0000","last_fullsized":"2024-09-21T10:34:41.226998+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:42.454658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":768,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857830+0000","last_change":"2024-09-21T10:25:07.652701+0000","last_active":"2024-09-21T10:34:44.857830+0000","last_peered":"2024-09-21T10:34:44.857830+0000","last_clean":"2024-09-21T10:34:44.857830+0000","last_became_active":"2024-09-21T10:24:24.825433+0000","last_became_peered":"2024-09-21T10:24:24.825433+0000","last_unstale":"2024-09-21T10:34:44.857830+0000","last_undegraded":"2024-09-21T10:34:44.857830+0000","last_fullsized":"2024-09-21T10:34:44.857830+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:25:07.652427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:25:07.652427+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:30.775517+0000","scrub_duration":0.024223608000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":981,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857109+0000","last_change":"2024-09-21T10:30:45.207431+0000","last_active":"2024-09-21T10:34:44.857109+0000","last_peered":"2024-09-21T10:34:44.857109+0000","last_clean":"2024-09-21T10:34:44.857109+0000","last_became_active":"2024-09-21T10:30:45.207075+0000","last_became_peered":"2024-09-21T10:30:45.207075+0000","last_unstale":"2024-09-21T10:34:44.857109+0000","last_undegraded":"2024-09-21T10:34:44.857109+0000","last_fullsized":"2024-09-21T10:34:44.857109+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:08.684578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":80,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938382+0000","last_change":"2024-09-21T10:33:21.933455+0000","last_active":"2024-09-21T10:34:44.938382+0000","last_peered":"2024-09-21T10:34:44.938382+0000","last_clean":"2024-09-21T10:34:44.938382+0000","last_became_active":"2024-09-21T10:33:21.933188+0000","last_became_peered":"2024-09-21T10:33:21.933188+0000","last_unstale":"2024-09-21T10:34:44.938382+0000","last_undegraded":"2024-09-21T10:34:44.938382+0000","last_fullsized":"2024-09-21T10:34:44.938382+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:38.403373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":838,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.477608+0000","last_change":"2024-09-21T10:34:29.176105+0000","last_active":"2024-09-21T10:34:40.477608+0000","last_peered":"2024-09-21T10:34:40.477608+0000","last_clean":"2024-09-21T10:34:40.477608+0000","last_became_active":"2024-09-21T10:34:29.175632+0000","last_became_peered":"2024-09-21T10:34:29.175632+0000","last_unstale":"2024-09-21T10:34:40.477608+0000","last_undegraded":"2024-09-21T10:34:40.477608+0000","last_fullsized":"2024-09-21T10:34:40.477608+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:32:45.775219+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:32:45.775219+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:15.888314+0000","scrub_duration":0.016223530999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'139","reported_seq":845,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858390+0000","last_change":"2024-09-21T10:24:17.810946+0000","last_active":"2024-09-21T10:34:44.858390+0000","last_peered":"2024-09-21T10:34:44.858390+0000","last_clean":"2024-09-21T10:34:44.858390+0000","last_became_active":"2024-09-21T10:24:17.810513+0000","last_became_peered":"2024-09-21T10:24:17.810513+0000","last_unstale":"2024-09-21T10:34:44.858390+0000","last_undegraded":"2024-09-21T10:34:44.858390+0000","last_fullsized":"2024-09-21T10:34:44.858390+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:06.614949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043433399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":393,"num_read_kb":630,"num_write":137,"num_write_kb":232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":899,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857381+0000","last_change":"2024-09-21T10:20:13.081658+0000","last_active":"2024-09-21T10:34:44.857381+0000","last_peered":"2024-09-21T10:34:44.857381+0000","last_clean":"2024-09-21T10:34:44.857381+0000","last_became_active":"2024-09-21T10:20:13.081260+0000","last_became_peered":"2024-09-21T10:20:13.081260+0000","last_unstale":"2024-09-21T10:34:44.857381+0000","last_undegraded":"2024-09-21T10:34:44.857381+0000","last_fullsized":"2024-09-21T10:34:44.857381+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:32.808726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":508,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906578+0000","last_change":"2024-09-21T10:19:50.551350+0000","last_active":"2024-09-21T10:34:40.906578+0000","last_peered":"2024-09-21T10:34:40.906578+0000","last_clean":"2024-09-21T10:34:40.906578+0000","last_became_active":"2024-09-21T10:19:50.550526+0000","last_became_peered":"2024-09-21T10:19:50.550526+0000","last_unstale":"2024-09-21T10:34:40.906578+0000","last_undegraded":"2024-09-21T10:34:40.906578+0000","last_fullsized":"2024-09-21T10:34:40.906578+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:48.074957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":777,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938037+0000","last_change":"2024-09-21T10:30:58.540050+0000","last_active":"2024-09-21T10:34:44.938037+0000","last_peered":"2024-09-21T10:34:44.938037+0000","last_clean":"2024-09-21T10:34:44.938037+0000","last_became_active":"2024-09-21T10:30:47.975931+0000","last_became_peered":"2024-09-21T10:30:47.975931+0000","last_unstale":"2024-09-21T10:34:44.938037+0000","last_undegraded":"2024-09-21T10:34:44.938037+0000","last_fullsized":"2024-09-21T10:34:44.938037+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:58.539975+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:19.450903+0000","last_clean_scrub_stamp":"2024-09-21T10:30:58.539975+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:02.759105+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'143","reported_seq":1367,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937393+0000","last_change":"2024-09-21T10:33:39.998744+0000","last_active":"2024-09-21T10:34:44.937393+0000","last_peered":"2024-09-21T10:34:44.937393+0000","last_clean":"2024-09-21T10:34:44.937393+0000","last_became_active":"2024-09-21T10:33:39.998073+0000","last_became_peered":"2024-09-21T10:33:39.998073+0000","last_unstale":"2024-09-21T10:34:44.937393+0000","last_undegraded":"2024-09-21T10:34:44.937393+0000","last_fullsized":"2024-09-21T10:34:44.937393+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":143,"log_dups_size":0,"ondisk_log_size":143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:17.802386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057787099999999998,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":780,"num_read_kb":2160,"num_write":122,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":33554432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937005+0000","last_change":"2024-09-21T10:33:37.004358+0000","last_active":"2024-09-21T10:34:44.937005+0000","last_peered":"2024-09-21T10:34:44.937005+0000","last_clean":"2024-09-21T10:34:44.937005+0000","last_became_active":"2024-09-21T10:33:37.003994+0000","last_became_peered":"2024-09-21T10:33:37.003994+0000","last_unstale":"2024-09-21T10:34:44.937005+0000","last_undegraded":"2024-09-21T10:34:44.937005+0000","last_fullsized":"2024-09-21T10:34:44.937005+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:43.298113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937293+0000","last_change":"2024-09-21T10:33:36.996808+0000","last_active":"2024-09-21T10:34:44.937293+0000","last_peered":"2024-09-21T10:34:44.937293+0000","last_clean":"2024-09-21T10:34:44.937293+0000","last_became_active":"2024-09-21T10:33:36.996451+0000","last_became_peered":"2024-09-21T10:33:36.996451+0000","last_unstale":"2024-09-21T10:34:44.937293+0000","last_undegraded":"2024-09-21T10:34:44.937293+0000","last_fullsized":"2024-09-21T10:34:44.937293+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:28:01.658066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":855,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.477839+0000","last_change":"2024-09-21T10:34:29.178210+0000","last_active":"2024-09-21T10:34:40.477839+0000","last_peered":"2024-09-21T10:34:40.477839+0000","last_clean":"2024-09-21T10:34:40.477839+0000","last_became_active":"2024-09-21T10:34:29.177782+0000","last_became_peered":"2024-09-21T10:34:29.177782+0000","last_unstale":"2024-09-21T10:34:40.477839+0000","last_undegraded":"2024-09-21T10:34:40.477839+0000","last_fullsized":"2024-09-21T10:34:40.477839+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:32:53.227153+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:32:53.227153+0000","last_clean_scrub_stamp":"2024-09-21T10:32:53.227153+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:24:21.865898+0000","scrub_duration":0.010887529,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'135","reported_seq":1567,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937395+0000","last_change":"2024-09-21T10:33:39.994395+0000","last_active":"2024-09-21T10:34:44.937395+0000","last_peered":"2024-09-21T10:34:44.937395+0000","last_clean":"2024-09-21T10:34:44.937395+0000","last_became_active":"2024-09-21T10:33:39.993621+0000","last_became_peered":"2024-09-21T10:33:39.993621+0000","last_unstale":"2024-09-21T10:34:44.937395+0000","last_undegraded":"2024-09-21T10:34:44.937395+0000","last_fullsized":"2024-09-21T10:34:44.937395+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":135,"log_dups_size":0,"ondisk_log_size":135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:22.740479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349966,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":800,"num_read_kb":913,"num_write":127,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937245+0000","last_change":"2024-09-21T10:30:45.211483+0000","last_active":"2024-09-21T10:34:44.937245+0000","last_peered":"2024-09-21T10:34:44.937245+0000","last_clean":"2024-09-21T10:34:44.937245+0000","last_became_active":"2024-09-21T10:30:45.211153+0000","last_became_peered":"2024-09-21T10:30:45.211153+0000","last_unstale":"2024-09-21T10:34:44.937245+0000","last_undegraded":"2024-09-21T10:34:44.937245+0000","last_fullsized":"2024-09-21T10:34:44.937245+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:47:17.004820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":200,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938094+0000","last_change":"2024-09-21T10:33:37.001489+0000","last_active":"2024-09-21T10:34:44.938094+0000","last_peered":"2024-09-21T10:34:44.938094+0000","last_clean":"2024-09-21T10:34:44.938094+0000","last_became_active":"2024-09-21T10:33:37.001116+0000","last_became_peered":"2024-09-21T10:33:37.001116+0000","last_unstale":"2024-09-21T10:34:44.938094+0000","last_undegraded":"2024-09-21T10:34:44.938094+0000","last_fullsized":"2024-09-21T10:34:44.938094+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:05:29.964124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":844,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858035+0000","last_change":"2024-09-21T10:33:39.992102+0000","last_active":"2024-09-21T10:34:44.858035+0000","last_peered":"2024-09-21T10:34:44.858035+0000","last_clean":"2024-09-21T10:34:44.858035+0000","last_became_active":"2024-09-21T10:33:39.991701+0000","last_became_peered":"2024-09-21T10:33:39.991701+0000","last_unstale":"2024-09-21T10:34:44.858035+0000","last_undegraded":"2024-09-21T10:34:44.858035+0000","last_fullsized":"2024-09-21T10:34:44.858035+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:22.717550+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-09-21T10:19:13.371825+0000","last_clean_scrub_stamp":"2024-09-21T10:30:22.717550+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:45.929549+0000","scrub_duration":0.0073004899999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'122","reported_seq":1362,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858151+0000","last_change":"2024-09-21T10:30:51.982312+0000","last_active":"2024-09-21T10:34:44.858151+0000","last_peered":"2024-09-21T10:34:44.858151+0000","last_clean":"2024-09-21T10:34:44.858151+0000","last_became_active":"2024-09-21T10:30:51.981405+0000","last_became_peered":"2024-09-21T10:30:51.981405+0000","last_unstale":"2024-09-21T10:34:44.858151+0000","last_undegraded":"2024-09-21T10:34:44.858151+0000","last_fullsized":"2024-09-21T10:34:44.858151+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:15.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059342900000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1155,"num_read_kb":1031,"num_write":91,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":268,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226127+0000","last_change":"2024-09-21T10:33:36.995672+0000","last_active":"2024-09-21T10:34:41.226127+0000","last_peered":"2024-09-21T10:34:41.226127+0000","last_clean":"2024-09-21T10:34:41.226127+0000","last_became_active":"2024-09-21T10:33:36.995193+0000","last_became_peered":"2024-09-21T10:33:36.995193+0000","last_unstale":"2024-09-21T10:34:41.226127+0000","last_undegraded":"2024-09-21T10:34:41.226127+0000","last_fullsized":"2024-09-21T10:34:41.226127+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:24.513124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938003+0000","last_change":"2024-09-21T10:30:45.210421+0000","last_active":"2024-09-21T10:34:44.938003+0000","last_peered":"2024-09-21T10:34:44.938003+0000","last_clean":"2024-09-21T10:34:44.938003+0000","last_became_active":"2024-09-21T10:30:45.206024+0000","last_became_peered":"2024-09-21T10:30:45.206024+0000","last_unstale":"2024-09-21T10:34:44.938003+0000","last_undegraded":"2024-09-21T10:34:44.938003+0000","last_fullsized":"2024-09-21T10:34:44.938003+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:51.844704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":671,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938180+0000","last_change":"2024-09-21T10:34:29.176018+0000","last_active":"2024-09-21T10:34:44.938180+0000","last_peered":"2024-09-21T10:34:44.938180+0000","last_clean":"2024-09-21T10:34:44.938180+0000","last_became_active":"2024-09-21T10:34:29.175414+0000","last_became_peered":"2024-09-21T10:34:29.175414+0000","last_unstale":"2024-09-21T10:34:44.938180+0000","last_undegraded":"2024-09-21T10:34:44.938180+0000","last_fullsized":"2024-09-21T10:34:44.938180+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:31:00.915274+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:31:00.915274+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:16.839294+0000","scrub_duration":0.0051508819999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":481,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857531+0000","last_change":"2024-09-21T10:20:13.084678+0000","last_active":"2024-09-21T10:34:44.857531+0000","last_peered":"2024-09-21T10:34:44.857531+0000","last_clean":"2024-09-21T10:34:44.857531+0000","last_became_active":"2024-09-21T10:20:13.084367+0000","last_became_peered":"2024-09-21T10:20:13.084367+0000","last_unstale":"2024-09-21T10:34:44.857531+0000","last_undegraded":"2024-09-21T10:34:44.857531+0000","last_fullsized":"2024-09-21T10:34:44.857531+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:01.388173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":983,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937770+0000","last_change":"2024-09-21T10:33:40.002608+0000","last_active":"2024-09-21T10:34:44.937770+0000","last_peered":"2024-09-21T10:34:44.937770+0000","last_clean":"2024-09-21T10:34:44.937770+0000","last_became_active":"2024-09-21T10:33:40.001927+0000","last_became_peered":"2024-09-21T10:33:40.001927+0000","last_unstale":"2024-09-21T10:34:44.937770+0000","last_undegraded":"2024-09-21T10:34:44.937770+0000","last_fullsized":"2024-09-21T10:34:44.937770+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:39:53.531362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022851600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":612,"num_read_kb":716,"num_write":117,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937439+0000","last_change":"2024-09-21T10:30:45.223025+0000","last_active":"2024-09-21T10:34:44.937439+0000","last_peered":"2024-09-21T10:34:44.937439+0000","last_clean":"2024-09-21T10:34:44.937439+0000","last_became_active":"2024-09-21T10:30:45.222585+0000","last_became_peered":"2024-09-21T10:30:45.222585+0000","last_unstale":"2024-09-21T10:34:44.937439+0000","last_undegraded":"2024-09-21T10:34:44.937439+0000","last_fullsized":"2024-09-21T10:34:44.937439+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:14.096008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":692,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938559+0000","last_change":"2024-09-21T10:33:39.998237+0000","last_active":"2024-09-21T10:34:44.938559+0000","last_peered":"2024-09-21T10:34:44.938559+0000","last_clean":"2024-09-21T10:34:44.938559+0000","last_became_active":"2024-09-21T10:33:39.997793+0000","last_became_peered":"2024-09-21T10:33:39.997793+0000","last_unstale":"2024-09-21T10:34:44.938559+0000","last_undegraded":"2024-09-21T10:34:44.938559+0000","last_fullsized":"2024-09-21T10:34:44.938559+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:26.906418+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:26.906418+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:36.945992+0000","scrub_duration":0.0087083520000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937905+0000","last_change":"2024-09-21T10:30:45.210705+0000","last_active":"2024-09-21T10:34:44.937905+0000","last_peered":"2024-09-21T10:34:44.937905+0000","last_clean":"2024-09-21T10:34:44.937905+0000","last_became_active":"2024-09-21T10:30:45.205528+0000","last_became_peered":"2024-09-21T10:30:45.205528+0000","last_unstale":"2024-09-21T10:34:44.937905+0000","last_undegraded":"2024-09-21T10:34:44.937905+0000","last_fullsized":"2024-09-21T10:34:44.937905+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:14:32.515942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'98","reported_seq":1174,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938223+0000","last_change":"2024-09-21T10:33:37.023942+0000","last_active":"2024-09-21T10:34:44.938223+0000","last_peered":"2024-09-21T10:34:44.938223+0000","last_clean":"2024-09-21T10:34:44.938223+0000","last_became_active":"2024-09-21T10:33:37.001270+0000","last_became_peered":"2024-09-21T10:33:37.001270+0000","last_unstale":"2024-09-21T10:34:44.938223+0000","last_undegraded":"2024-09-21T10:34:44.938223+0000","last_fullsized":"2024-09-21T10:34:44.938223+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:16:01.086134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081177500000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":788,"num_read_kb":1685,"num_write":73,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937209+0000","last_change":"2024-09-21T10:33:36.999948+0000","last_active":"2024-09-21T10:34:44.937209+0000","last_peered":"2024-09-21T10:34:44.937209+0000","last_clean":"2024-09-21T10:34:44.937209+0000","last_became_active":"2024-09-21T10:33:36.999416+0000","last_became_peered":"2024-09-21T10:33:36.999416+0000","last_unstale":"2024-09-21T10:34:44.937209+0000","last_undegraded":"2024-09-21T10:34:44.937209+0000","last_fullsized":"2024-09-21T10:34:44.937209+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:15:33.790607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":789,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858203+0000","last_change":"2024-09-21T10:30:22.584185+0000","last_active":"2024-09-21T10:34:44.858203+0000","last_peered":"2024-09-21T10:34:44.858203+0000","last_clean":"2024-09-21T10:34:44.858203+0000","last_became_active":"2024-09-21T10:30:22.056768+0000","last_became_peered":"2024-09-21T10:30:22.056768+0000","last_unstale":"2024-09-21T10:34:44.858203+0000","last_undegraded":"2024-09-21T10:34:44.858203+0000","last_fullsized":"2024-09-21T10:34:44.858203+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-09-21T10:30:22.583825+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:46:40.454871+0000","last_clean_scrub_stamp":"2024-09-21T10:30:22.583825+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:59.427627+0000","scrub_duration":0.0051540479999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1009,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857919+0000","last_change":"2024-09-21T10:33:37.012499+0000","last_active":"2024-09-21T10:34:44.857919+0000","last_peered":"2024-09-21T10:34:44.857919+0000","last_clean":"2024-09-21T10:34:44.857919+0000","last_became_active":"2024-09-21T10:33:36.998155+0000","last_became_peered":"2024-09-21T10:33:36.998155+0000","last_unstale":"2024-09-21T10:34:44.857919+0000","last_undegraded":"2024-09-21T10:34:44.857919+0000","last_fullsized":"2024-09-21T10:34:44.857919+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:46.183541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'169","reported_seq":841,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.938238+0000","last_change":"2024-09-21T10:30:51.986197+0000","last_active":"2024-09-21T10:34:44.938238+0000","last_peered":"2024-09-21T10:34:44.938238+0000","last_clean":"2024-09-21T10:34:44.938238+0000","last_became_active":"2024-09-21T10:30:51.985466+0000","last_became_peered":"2024-09-21T10:30:51.985466+0000","last_unstale":"2024-09-21T10:34:44.938238+0000","last_undegraded":"2024-09-21T10:34:44.938238+0000","last_fullsized":"2024-09-21T10:34:44.938238+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:16.999338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086462099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1040,"num_read_kb":3301,"num_write":145,"num_write_kb":2329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1011,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857046+0000","last_change":"2024-09-21T10:34:26.158778+0000","last_active":"2024-09-21T10:34:44.857046+0000","last_peered":"2024-09-21T10:34:44.857046+0000","last_clean":"2024-09-21T10:34:44.857046+0000","last_became_active":"2024-09-21T10:34:26.158428+0000","last_became_peered":"2024-09-21T10:34:26.158428+0000","last_unstale":"2024-09-21T10:34:44.857046+0000","last_undegraded":"2024-09-21T10:34:44.857046+0000","last_fullsized":"2024-09-21T10:34:44.857046+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:08:54.150856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"373'150","reported_seq":970,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857542+0000","last_change":"2024-09-21T10:30:22.757965+0000","last_active":"2024-09-21T10:34:44.857542+0000","last_peered":"2024-09-21T10:34:44.857542+0000","last_clean":"2024-09-21T10:34:44.857542+0000","last_became_active":"2024-09-21T10:30:22.041873+0000","last_became_peered":"2024-09-21T10:30:22.041873+0000","last_unstale":"2024-09-21T10:34:44.857542+0000","last_undegraded":"2024-09-21T10:34:44.857542+0000","last_fullsized":"2024-09-21T10:34:44.857542+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":290,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":3,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:30:22.757772+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:30:22.757772+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:17.470049+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":66,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937510+0000","last_change":"2024-09-21T10:33:37.013764+0000","last_active":"2024-09-21T10:34:44.937510+0000","last_peered":"2024-09-21T10:34:44.937510+0000","last_clean":"2024-09-21T10:34:44.937510+0000","last_became_active":"2024-09-21T10:33:37.001151+0000","last_became_peered":"2024-09-21T10:33:37.001151+0000","last_unstale":"2024-09-21T10:34:44.937510+0000","last_undegraded":"2024-09-21T10:34:44.937510+0000","last_fullsized":"2024-09-21T10:34:44.937510+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:39:51.809582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'52","reported_seq":905,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.694915+0000","last_change":"2024-09-21T10:34:26.170165+0000","last_active":"2024-09-21T10:34:44.694915+0000","last_peered":"2024-09-21T10:34:44.694915+0000","last_clean":"2024-09-21T10:34:44.694915+0000","last_became_active":"2024-09-21T10:34:26.156140+0000","last_became_peered":"2024-09-21T10:34:26.156140+0000","last_unstale":"2024-09-21T10:34:44.694915+0000","last_undegraded":"2024-09-21T10:34:44.694915+0000","last_fullsized":"2024-09-21T10:34:44.694915+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:49:09.161494+0000","last_clean_scrub_stamp":"2024-09-21T09:49:09.161494+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:06.763124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024593500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":2574,"num_write":34,"num_write_kb":3164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":505,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226555+0000","last_change":"2024-09-21T10:20:14.263598+0000","last_active":"2024-09-21T10:34:41.226555+0000","last_peered":"2024-09-21T10:34:41.226555+0000","last_clean":"2024-09-21T10:34:41.226555+0000","last_became_active":"2024-09-21T10:20:14.263340+0000","last_became_peered":"2024-09-21T10:20:14.263340+0000","last_unstale":"2024-09-21T10:34:41.226555+0000","last_undegraded":"2024-09-21T10:34:41.226555+0000","last_fullsized":"2024-09-21T10:34:41.226555+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.805005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":547,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857813+0000","last_change":"2024-09-21T10:30:45.220621+0000","last_active":"2024-09-21T10:34:44.857813+0000","last_peered":"2024-09-21T10:34:44.857813+0000","last_clean":"2024-09-21T10:34:44.857813+0000","last_became_active":"2024-09-21T10:30:45.219593+0000","last_became_peered":"2024-09-21T10:30:45.219593+0000","last_unstale":"2024-09-21T10:34:44.857813+0000","last_undegraded":"2024-09-21T10:34:44.857813+0000","last_fullsized":"2024-09-21T10:34:44.857813+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:27:44.914386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":238,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226696+0000","last_change":"2024-09-21T10:19:50.589068+0000","last_active":"2024-09-21T10:34:41.226696+0000","last_peered":"2024-09-21T10:34:41.226696+0000","last_clean":"2024-09-21T10:34:41.226696+0000","last_became_active":"2024-09-21T10:19:50.551835+0000","last_became_peered":"2024-09-21T10:19:50.551835+0000","last_unstale":"2024-09-21T10:34:41.226696+0000","last_undegraded":"2024-09-21T10:34:41.226696+0000","last_fullsized":"2024-09-21T10:34:41.226696+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:45.811615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":182,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937590+0000","last_change":"2024-09-21T10:30:45.217613+0000","last_active":"2024-09-21T10:34:44.937590+0000","last_peered":"2024-09-21T10:34:44.937590+0000","last_clean":"2024-09-21T10:34:44.937590+0000","last_became_active":"2024-09-21T10:30:45.217188+0000","last_became_peered":"2024-09-21T10:30:45.217188+0000","last_unstale":"2024-09-21T10:34:44.937590+0000","last_undegraded":"2024-09-21T10:34:44.937590+0000","last_fullsized":"2024-09-21T10:34:44.937590+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:54.177397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":24,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.478599+0000","last_change":"2024-09-21T10:34:26.164361+0000","last_active":"2024-09-21T10:34:40.478599+0000","last_peered":"2024-09-21T10:34:40.478599+0000","last_clean":"2024-09-21T10:34:40.478599+0000","last_became_active":"2024-09-21T10:34:26.159487+0000","last_became_peered":"2024-09-21T10:34:26.159487+0000","last_unstale":"2024-09-21T10:34:40.478599+0000","last_undegraded":"2024-09-21T10:34:40.478599+0000","last_fullsized":"2024-09-21T10:34:40.478599+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:34:19.301610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"373'150","reported_seq":973,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937785+0000","last_change":"2024-09-21T10:30:31.115487+0000","last_active":"2024-09-21T10:34:44.937785+0000","last_peered":"2024-09-21T10:34:44.937785+0000","last_clean":"2024-09-21T10:34:44.937785+0000","last_became_active":"2024-09-21T10:30:31.114941+0000","last_became_peered":"2024-09-21T10:30:31.114941+0000","last_unstale":"2024-09-21T10:34:44.937785+0000","last_undegraded":"2024-09-21T10:34:44.937785+0000","last_fullsized":"2024-09-21T10:34:44.937785+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":430,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"373'150","last_scrub_stamp":"2024-09-21T10:30:23.771955+0000","last_deep_scrub":"373'150","last_deep_scrub_stamp":"2024-09-21T10:24:18.230314+0000","last_clean_scrub_stamp":"2024-09-21T10:30:23.771955+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:20:42.604348+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":258,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.353704+0000","last_change":"2024-09-21T10:30:45.199098+0000","last_active":"2024-09-21T10:34:41.353704+0000","last_peered":"2024-09-21T10:34:41.353704+0000","last_clean":"2024-09-21T10:34:41.353704+0000","last_became_active":"2024-09-21T10:30:45.198762+0000","last_became_peered":"2024-09-21T10:30:45.198762+0000","last_unstale":"2024-09-21T10:34:41.353704+0000","last_undegraded":"2024-09-21T10:34:41.353704+0000","last_fullsized":"2024-09-21T10:34:41.353704+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:40.566015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":905,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857123+0000","last_change":"2024-09-21T10:34:26.158771+0000","last_active":"2024-09-21T10:34:44.857123+0000","last_peered":"2024-09-21T10:34:44.857123+0000","last_clean":"2024-09-21T10:34:44.857123+0000","last_became_active":"2024-09-21T10:34:26.158381+0000","last_became_peered":"2024-09-21T10:34:26.158381+0000","last_unstale":"2024-09-21T10:34:44.857123+0000","last_undegraded":"2024-09-21T10:34:44.857123+0000","last_fullsized":"2024-09-21T10:34:44.857123+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:23.238104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937565+0000","last_change":"2024-09-21T10:33:37.005046+0000","last_active":"2024-09-21T10:34:44.937565+0000","last_peered":"2024-09-21T10:34:44.937565+0000","last_clean":"2024-09-21T10:34:44.937565+0000","last_became_active":"2024-09-21T10:33:37.004704+0000","last_became_peered":"2024-09-21T10:33:37.004704+0000","last_unstale":"2024-09-21T10:34:44.937565+0000","last_undegraded":"2024-09-21T10:34:44.937565+0000","last_fullsized":"2024-09-21T10:34:44.937565+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:36.930972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":972,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.856996+0000","last_change":"2024-09-21T10:33:22.277354+0000","last_active":"2024-09-21T10:34:44.856996+0000","last_peered":"2024-09-21T10:34:44.856996+0000","last_clean":"2024-09-21T10:34:44.856996+0000","last_became_active":"2024-09-21T10:33:22.276984+0000","last_became_peered":"2024-09-21T10:33:22.276984+0000","last_unstale":"2024-09-21T10:34:44.856996+0000","last_undegraded":"2024-09-21T10:34:44.856996+0000","last_fullsized":"2024-09-21T10:34:44.856996+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:10:43.527258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":565,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:40.906776+0000","last_change":"2024-09-21T10:30:45.209630+0000","last_active":"2024-09-21T10:34:40.906776+0000","last_peered":"2024-09-21T10:34:40.906776+0000","last_clean":"2024-09-21T10:34:40.906776+0000","last_became_active":"2024-09-21T10:30:45.209304+0000","last_became_peered":"2024-09-21T10:30:45.209304+0000","last_unstale":"2024-09-21T10:34:40.906776+0000","last_undegraded":"2024-09-21T10:34:40.906776+0000","last_fullsized":"2024-09-21T10:34:40.906776+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:29.693394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1011,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857118+0000","last_change":"2024-09-21T10:33:36.998197+0000","last_active":"2024-09-21T10:34:44.857118+0000","last_peered":"2024-09-21T10:34:44.857118+0000","last_clean":"2024-09-21T10:34:44.857118+0000","last_became_active":"2024-09-21T10:33:36.997830+0000","last_became_peered":"2024-09-21T10:33:36.997830+0000","last_unstale":"2024-09-21T10:34:44.857118+0000","last_undegraded":"2024-09-21T10:34:44.857118+0000","last_fullsized":"2024-09-21T10:34:44.857118+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:27.718538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":69,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937398+0000","last_change":"2024-09-21T10:34:26.161955+0000","last_active":"2024-09-21T10:34:44.937398+0000","last_peered":"2024-09-21T10:34:44.937398+0000","last_clean":"2024-09-21T10:34:44.937398+0000","last_became_active":"2024-09-21T10:34:26.161618+0000","last_became_peered":"2024-09-21T10:34:26.161618+0000","last_unstale":"2024-09-21T10:34:44.937398+0000","last_undegraded":"2024-09-21T10:34:44.937398+0000","last_fullsized":"2024-09-21T10:34:44.937398+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:53:04.405320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":181,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937729+0000","last_change":"2024-09-21T10:30:45.206647+0000","last_active":"2024-09-21T10:34:44.937729+0000","last_peered":"2024-09-21T10:34:44.937729+0000","last_clean":"2024-09-21T10:34:44.937729+0000","last_became_active":"2024-09-21T10:30:45.205723+0000","last_became_peered":"2024-09-21T10:30:45.205723+0000","last_unstale":"2024-09-21T10:34:44.937729+0000","last_undegraded":"2024-09-21T10:34:44.937729+0000","last_fullsized":"2024-09-21T10:34:44.937729+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:55:33.281876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":971,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857541+0000","last_change":"2024-09-21T10:30:45.207518+0000","last_active":"2024-09-21T10:34:44.857541+0000","last_peered":"2024-09-21T10:34:44.857541+0000","last_clean":"2024-09-21T10:34:44.857541+0000","last_became_active":"2024-09-21T10:30:45.207075+0000","last_became_peered":"2024-09-21T10:30:45.207075+0000","last_unstale":"2024-09-21T10:34:44.857541+0000","last_undegraded":"2024-09-21T10:34:44.857541+0000","last_fullsized":"2024-09-21T10:34:44.857541+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:33.239764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":554,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857749+0000","last_change":"2024-09-21T10:30:45.222126+0000","last_active":"2024-09-21T10:34:44.857749+0000","last_peered":"2024-09-21T10:34:44.857749+0000","last_clean":"2024-09-21T10:34:44.857749+0000","last_became_active":"2024-09-21T10:30:45.221747+0000","last_became_peered":"2024-09-21T10:30:45.221747+0000","last_unstale":"2024-09-21T10:34:44.857749+0000","last_undegraded":"2024-09-21T10:34:44.857749+0000","last_fullsized":"2024-09-21T10:34:44.857749+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:15.856857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":970,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937894+0000","last_change":"2024-09-21T10:30:45.216426+0000","last_active":"2024-09-21T10:34:44.937894+0000","last_peered":"2024-09-21T10:34:44.937894+0000","last_clean":"2024-09-21T10:34:44.937894+0000","last_became_active":"2024-09-21T10:30:45.215768+0000","last_became_peered":"2024-09-21T10:30:45.215768+0000","last_unstale":"2024-09-21T10:34:44.937894+0000","last_undegraded":"2024-09-21T10:34:44.937894+0000","last_fullsized":"2024-09-21T10:34:44.937894+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:13.389909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":62,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937097+0000","last_change":"2024-09-21T10:33:37.001740+0000","last_active":"2024-09-21T10:34:44.937097+0000","last_peered":"2024-09-21T10:34:44.937097+0000","last_clean":"2024-09-21T10:34:44.937097+0000","last_became_active":"2024-09-21T10:33:37.001289+0000","last_became_peered":"2024-09-21T10:33:37.001289+0000","last_unstale":"2024-09-21T10:34:44.937097+0000","last_undegraded":"2024-09-21T10:34:44.937097+0000","last_fullsized":"2024-09-21T10:34:44.937097+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:25:13.491213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":255,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857588+0000","last_change":"2024-09-21T10:34:22.335483+0000","last_active":"2024-09-21T10:34:44.857588+0000","last_peered":"2024-09-21T10:34:44.857588+0000","last_clean":"2024-09-21T10:34:44.857588+0000","last_became_active":"2024-09-21T10:34:22.335082+0000","last_became_peered":"2024-09-21T10:34:22.335082+0000","last_unstale":"2024-09-21T10:34:44.857588+0000","last_undegraded":"2024-09-21T10:34:44.857588+0000","last_fullsized":"2024-09-21T10:34:44.857588+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:10:48.483607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":498,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858342+0000","last_change":"2024-09-21T10:30:45.220768+0000","last_active":"2024-09-21T10:34:44.858342+0000","last_peered":"2024-09-21T10:34:44.858342+0000","last_clean":"2024-09-21T10:34:44.858342+0000","last_became_active":"2024-09-21T10:30:45.214059+0000","last_became_peered":"2024-09-21T10:30:45.214059+0000","last_unstale":"2024-09-21T10:34:44.858342+0000","last_undegraded":"2024-09-21T10:34:44.858342+0000","last_fullsized":"2024-09-21T10:34:44.858342+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:12:52.858780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":267,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857250+0000","last_change":"2024-09-21T10:33:36.995401+0000","last_active":"2024-09-21T10:34:44.857250+0000","last_peered":"2024-09-21T10:34:44.857250+0000","last_clean":"2024-09-21T10:34:44.857250+0000","last_became_active":"2024-09-21T10:33:36.995022+0000","last_became_peered":"2024-09-21T10:33:36.995022+0000","last_unstale":"2024-09-21T10:34:44.857250+0000","last_undegraded":"2024-09-21T10:34:44.857250+0000","last_fullsized":"2024-09-21T10:34:44.857250+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:15.343376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":450,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.858602+0000","last_change":"2024-09-21T10:20:14.272156+0000","last_active":"2024-09-21T10:34:44.858602+0000","last_peered":"2024-09-21T10:34:44.858602+0000","last_clean":"2024-09-21T10:34:44.858602+0000","last_became_active":"2024-09-21T10:20:14.271729+0000","last_became_peered":"2024-09-21T10:20:14.271729+0000","last_unstale":"2024-09-21T10:34:44.858602+0000","last_undegraded":"2024-09-21T10:34:44.858602+0000","last_fullsized":"2024-09-21T10:34:44.858602+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:00.126878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1000,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857467+0000","last_change":"2024-09-21T10:33:36.996843+0000","last_active":"2024-09-21T10:34:44.857467+0000","last_peered":"2024-09-21T10:34:44.857467+0000","last_clean":"2024-09-21T10:34:44.857467+0000","last_became_active":"2024-09-21T10:33:36.996479+0000","last_became_peered":"2024-09-21T10:33:36.996479+0000","last_unstale":"2024-09-21T10:34:44.857467+0000","last_undegraded":"2024-09-21T10:34:44.857467+0000","last_fullsized":"2024-09-21T10:34:44.857467+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:09:56.739587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":933,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.857762+0000","last_change":"2024-09-21T10:03:53.842870+0000","last_active":"2024-09-21T10:34:44.857762+0000","last_peered":"2024-09-21T10:34:44.857762+0000","last_clean":"2024-09-21T10:34:44.857762+0000","last_became_active":"2024-09-21T10:03:53.842295+0000","last_became_peered":"2024-09-21T10:03:53.842295+0000","last_unstale":"2024-09-21T10:34:44.857762+0000","last_undegraded":"2024-09-21T10:34:44.857762+0000","last_fullsized":"2024-09-21T10:34:44.857762+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:15.542781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":246,"reported_epoch":527,"state":"active+clean","last_fresh":"2024-09-21T10:34:41.226211+0000","last_change":"2024-09-21T10:30:45.206962+0000","last_active":"2024-09-21T10:34:41.226211+0000","last_peered":"2024-09-21T10:34:41.226211+0000","last_clean":"2024-09-21T10:34:41.226211+0000","last_became_active":"2024-09-21T10:30:45.206174+0000","last_became_peered":"2024-09-21T10:30:45.206174+0000","last_unstale":"2024-09-21T10:34:41.226211+0000","last_undegraded":"2024-09-21T10:34:41.226211+0000","last_fullsized":"2024-09-21T10:34:41.226211+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:22.381258+0000","last_clean_scrub_stamp":"2024-09-21T09:50:22.381258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:03:00.246788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":68,"reported_epoch":528,"state":"active+clean","last_fresh":"2024-09-21T10:34:44.937446+0000","last_change":"2024-09-21T10:33:37.011028+0000","last_active":"2024-09-21T10:34:44.937446+0000","last_peered":"2024-09-21T10:34:44.937446+0000","last_clean":"2024-09-21T10:34:44.937446+0000","last_became_active":"2024-09-21T10:33:36.998998+0000","last_became_peered":"2024-09-21T10:33:36.998998+0000","last_unstale":"2024-09-21T10:34:44.937446+0000","last_undegraded":"2024-09-21T10:34:44.937446+0000","last_fullsized":"2024-09-21T10:34:44.937446+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:50:21.379647+0000","last_clean_scrub_stamp":"2024-09-21T09:50:21.379647+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:01.037536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":6644,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30003,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5997,"num_read_kb":13010,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":376,"num_bytes_recovered":33554489,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3133,"ondisk_log_size":3133,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234552,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":958188,"kb_used_data":10720,"kb_used_omap":1,"kb_used_meta":947454,"kb_avail":92786964,"statfs":{"total":95995035648,"available":95013851136,"internally_reserved":0,"allocated":10977280,"data_stored":8366461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":970193355},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496190,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904360,"kb_used_data":6940,"kb_used_omap":1,"kb_used_meta":897406,"kb_avail":92840792,"statfs":{"total":95995035648,"available":95068971008,"internally_reserved":0,"allocated":7106560,"data_stored":4500041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":918944199},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":324,"seq":1391569404089,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":658212,"kb_used_data":6872,"kb_used_omap":7,"kb_used_meta":651320,"kb_avail":93086940,"statfs":{"total":95995035648,"available":95321026560,"internally_reserved":0,"allocated":7036928,"data_stored":4430875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7272,"internal_metadata":666952600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019463,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":876140,"kb_used_data":6624,"kb_used_omap":1,"kb_used_meta":869502,"kb_avail":92869012,"statfs":{"total":95995035648,"available":95097868288,"internally_reserved":0,"allocated":6782976,"data_stored":4172418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":890370502},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435647,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1159644,"kb_used_data":7184,"kb_used_omap":5,"kb_used_meta":1152442,"kb_avail":92585508,"statfs":{"total":95995035648,"available":94807560192,"internally_reserved":0,"allocated":7356416,"data_stored":4757986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6104,"internal_metadata":1180100648},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599174,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":949276,"kb_used_data":6864,"kb_used_omap":7,"kb_used_meta":942392,"kb_avail":92795876,"statfs":{"total":95995035648,"available":95022977024,"internally_reserved":0,"allocated":7028736,"data_stored":4430524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7731,"internal_metadata":965009869},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762701,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":940628,"kb_used_data":10696,"kb_used_omap":7,"kb_used_meta":929912,"kb_avail":92804524,"statfs":{"total":95995035648,"available":95031832576,"internally_reserved":0,"allocated":10952704,"data_stored":8361578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7939,"internal_metadata":952230141},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":491,"seq":2108828942366,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":78364,"kb_used_data":11024,"kb_used_omap":16,"kb_used_meta":67311,"kb_avail":93666788,"statfs":{"total":95995035648,"available":95914790912,"internally_reserved":0,"allocated":11288576,"data_stored":8690116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16669,"internal_metadata":68927203},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3521,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1334,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12186,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6345,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2616,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3178,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5678,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:35:35.797 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:35:35.797 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi032 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T10:35:35.971 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=1a404722ce8d4daeb2ea6b833e4a701d Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi032 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T10:35:35.974 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T10:35:35.983 INFO:tasks.cephadm:Teardown begin 2024-09-21T10:35:35.983 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:35:36.032 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:35:36.075 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T10:35:36.076 DEBUG:teuthology.orchestra.run.smithi032:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:35:36.094 DEBUG:teuthology.orchestra.run.smithi045:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:35:36.140 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T10:35:36.140 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-21T10:35:36.140 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a 2024-09-21T10:35:36.284 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:36 smithi045 ceph-mon[95976]: pgmap v1619: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:36.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:36 smithi045 ceph-mon[95976]: from='client.46670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:35:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-mon[101299]: pgmap v1619: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:36.457 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-mon[101299]: from='client.46670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:35:36.458 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:36 smithi032 systemd[1]: Stopping Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:35:36.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-mon[105748]: pgmap v1619: 93 pgs: 93 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:35:36.458 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-mon[105748]: from='client.46670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:35:36.861 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:36] ENGINE Bus STOPPING 2024-09-21T10:35:36.861 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T10:35:36.861 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:36] ENGINE Bus STOPPED 2024-09-21T10:35:36.861 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:36] ENGINE Bus STARTING 2024-09-21T10:35:36.861 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:36] ENGINE Serving on http://:::9283 2024-09-21T10:35:36.861 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:36] ENGINE Bus STARTED 2024-09-21T10:35:36.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:35:36.477+0000 7f5331b60640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:35:36.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:36 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a[101295]: 2024-09-21T10:35:36.477+0000 7f5331b60640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-21T10:35:36.862 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:36 smithi032 podman[333710]: 2024-09-21 10:35:36.550305456 +0000 UTC m=+0.217329185 container died 525e81f3c17e3af3fbb410b4ee38f2dd0968b23a35661aea237358d815faa740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-21T10:35:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:36 smithi032 podman[333710]: 2024-09-21 10:35:36.968831303 +0000 UTC m=+0.635855034 container cleanup 525e81f3c17e3af3fbb410b4ee38f2dd0968b23a35661aea237358d815faa740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-21T10:35:37.207 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:36 smithi032 bash[333710]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a 2024-09-21T10:35:37.454 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service' 2024-09-21T10:35:37.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:37 smithi032 podman[333721]: 2024-09-21 10:35:37.220274847 +0000 UTC m=+0.670593749 container remove 525e81f3c17e3af3fbb410b4ee38f2dd0968b23a35661aea237358d815faa740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T10:35:37.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:37 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T10:35:37.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:37 smithi032 systemd[1]: Stopped Ceph mon.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:35:37.477 INFO:journalctl@ceph.mon.a.smithi032.stdout:Sep 21 10:35:37 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.a.service: Consumed 36.070s CPU time. 2024-09-21T10:35:37.493 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:35:37.494 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-21T10:35:37.494 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-21T10:35:37.494 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.c 2024-09-21T10:35:37.825 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:37 smithi032 systemd[1]: Stopping Ceph mon.c for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:35:38.150 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:37] ENGINE Bus STOPPING 2024-09-21T10:35:38.150 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c[105744]: 2024-09-21T10:35:37.822+0000 7f260314e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:35:38.150 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:37 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c[105744]: 2024-09-21T10:35:37.822+0000 7f260314e640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-21T10:35:38.150 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:37 smithi032 podman[333843]: 2024-09-21 10:35:37.874729605 +0000 UTC m=+0.155382993 container died 233d45e2730d79fdf72f36f6405f3d5b5b06b13a52c98cb5cd6c2cee0fb44a94 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c, RELEASE=main-62da54b, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T10:35:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:38 smithi032 podman[333843]: 2024-09-21 10:35:38.374162402 +0000 UTC m=+0.654815845 container cleanup 233d45e2730d79fdf72f36f6405f3d5b5b06b13a52c98cb5cd6c2cee0fb44a94 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c, RELEASE=main-62da54b, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:35:38.457 INFO:journalctl@ceph.mon.c.smithi032.stdout:Sep 21 10:35:38 smithi032 bash[333843]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-c 2024-09-21T10:35:38.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T10:35:38.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:38] ENGINE Bus STOPPED 2024-09-21T10:35:38.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:38] ENGINE Bus STARTING 2024-09-21T10:35:38.457 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:38] ENGINE Serving on http://:::9283 2024-09-21T10:35:38.458 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:38 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: [21/Sep/2024:10:35:38] ENGINE Bus STARTED 2024-09-21T10:35:38.819 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.c.service' 2024-09-21T10:35:38.854 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:35:38.854 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-21T10:35:38.855 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-21T10:35:38.855 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.b 2024-09-21T10:35:39.285 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:38 smithi045 systemd[1]: Stopping Ceph mon.b for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:35:39.957 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:39 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: ::ffff:172.21.15.45 - - [21/Sep/2024:10:35:39] "GET /metrics HTTP/1.1" 200 34134 "" "Prometheus/2.51.0" 2024-09-21T10:35:40.470 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b[95972]: 2024-09-21T10:35:40.142+0000 7f50a6ae1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:35:40.470 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b[95972]: 2024-09-21T10:35:40.142+0000 7f50a6ae1640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-21T10:35:40.471 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:40 smithi045 podman[238960]: 2024-09-21 10:35:40.173068207 +0000 UTC m=+0.165544564 container died 7115dfb7957aa655f8d4a64b56d1b2b4277b6b6bebdae068d37f17a0744714c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T10:35:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:40 smithi045 podman[238960]: 2024-09-21 10:35:40.675425076 +0000 UTC m=+0.667901425 container cleanup 7115dfb7957aa655f8d4a64b56d1b2b4277b6b6bebdae068d37f17a0744714c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:35:40.785 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:40 smithi045 bash[238960]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b 2024-09-21T10:35:41.264 INFO:journalctl@ceph.mon.b.smithi045.stdout:Sep 21 10:35:40 smithi045 podman[238997]: 2024-09-21 10:35:40.885288275 +0000 UTC m=+0.709456989 container remove 7115dfb7957aa655f8d4a64b56d1b2b4277b6b6bebdae068d37f17a0744714c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-62da54b) 2024-09-21T10:35:41.575 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mon.b.service' 2024-09-21T10:35:41.639 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:35:41.640 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-21T10:35:41.640 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-21T10:35:41.640 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y 2024-09-21T10:35:41.956 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:41 smithi032 systemd[1]: Stopping Ceph mgr.y for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:35:42.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:41 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y[85228]: 2024-09-21T10:35:41.953+0000 7fced3ccc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:35:42.207 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:41 smithi032 podman[333976]: 2024-09-21 10:35:41.995958184 +0000 UTC m=+0.154967411 container died 581ec7ea9f79414c143dd4fa322421d2e6fc5d11699b272c86eff89c842de324 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T10:35:42.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:42 smithi032 podman[333976]: 2024-09-21 10:35:42.562762651 +0000 UTC m=+0.721771876 container cleanup 581ec7ea9f79414c143dd4fa322421d2e6fc5d11699b272c86eff89c842de324 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T10:35:42.707 INFO:journalctl@ceph.mgr.y.smithi032.stdout:Sep 21 10:35:42 smithi032 bash[333976]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-y 2024-09-21T10:35:43.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:35:42 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:35:42.570Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T10:35:43.059 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.y.service' 2024-09-21T10:35:43.094 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:35:43.095 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-21T10:35:43.095 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-21T10:35:43.095 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x 2024-09-21T10:35:43.434 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 10:35:43 smithi045 systemd[1]: Stopping Ceph mgr.x for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:35:43.703 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 10:35:43 smithi045 podman[239149]: 2024-09-21 10:35:43.432086106 +0000 UTC m=+0.127017403 container died 7ecdc05240855f70b1581bbc7751e684bd0be5e6f3002f472736f315f0f52592 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:35:44.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 10:35:43 smithi045 podman[239149]: 2024-09-21 10:35:43.842356252 +0000 UTC m=+0.537287538 container cleanup 7ecdc05240855f70b1581bbc7751e684bd0be5e6f3002f472736f315f0f52592 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T10:35:44.035 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 10:35:43 smithi045 bash[239149]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x 2024-09-21T10:35:44.388 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 10:35:44 smithi045 podman[239160]: 2024-09-21 10:35:44.127499079 +0000 UTC m=+0.696610220 container remove 7ecdc05240855f70b1581bbc7751e684bd0be5e6f3002f472736f315f0f52592 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-mgr-x, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T10:35:44.388 INFO:journalctl@ceph.mgr.x.smithi045.stdout:Sep 21 10:35:44 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:35:44.392 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@mgr.x.service' 2024-09-21T10:35:44.428 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:35:44.428 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-21T10:35:44.428 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T10:35:44.429 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:35:44.728 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:44 smithi032 systemd[1]: Stopping Ceph osd.0 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:35:45.193 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[319501]: 2024-09-21T10:35:44.724+0000 7f876ee1d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:35:45.193 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[319501]: 2024-09-21T10:35:44.724+0000 7f876ee1d640 -1 osd.0 528 *** Got signal Terminated *** 2024-09-21T10:35:45.193 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:44 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0[319501]: 2024-09-21T10:35:44.724+0000 7f876ee1d640 -1 osd.0 528 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:35:50.457 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:50 smithi032 podman[334100]: 2024-09-21 10:35:50.072677093 +0000 UTC m=+5.444441906 container died f88bf0350cada02bc5dc6933b8ee1f54abfde15f8fbab526fff2f1c832c166a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T10:35:50.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:50 smithi032 podman[334100]: 2024-09-21 10:35:50.492478965 +0000 UTC m=+5.864243758 container cleanup f88bf0350cada02bc5dc6933b8ee1f54abfde15f8fbab526fff2f1c832c166a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T10:35:50.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:50 smithi032 bash[334100]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:35:50.957 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:50 smithi032 podman[334115]: 2024-09-21 10:35:50.685555374 +0000 UTC m=+0.615482875 container remove f88bf0350cada02bc5dc6933b8ee1f54abfde15f8fbab526fff2f1c832c166a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:35:51.424 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:51 smithi032 podman[334207]: 2024-09-21 10:35:51.032457783 +0000 UTC m=+0.016329261 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:35:51.424 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:51 smithi032 podman[334207]: 2024-09-21 10:35:51.136363315 +0000 UTC m=+0.120234783 container create 3d5db8fbb94c45fb074b3c4aec78730314b9c151e1e347d96eae687fd3db39b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T10:35:51.424 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:51 smithi032 podman[334207]: 2024-09-21 10:35:51.422481191 +0000 UTC m=+0.406352660 container init 3d5db8fbb94c45fb074b3c4aec78730314b9c151e1e347d96eae687fd3db39b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T10:35:51.707 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:51 smithi032 podman[334207]: 2024-09-21 10:35:51.426091197 +0000 UTC m=+0.409962666 container start 3d5db8fbb94c45fb074b3c4aec78730314b9c151e1e347d96eae687fd3db39b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T10:35:51.708 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:51 smithi032 podman[334207]: 2024-09-21 10:35:51.539302194 +0000 UTC m=+0.523173664 container attach 3d5db8fbb94c45fb074b3c4aec78730314b9c151e1e347d96eae687fd3db39b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, CEPH_POINT_RELEASE=) 2024-09-21T10:35:52.069 INFO:journalctl@ceph.osd.0.smithi032.stdout:Sep 21 10:35:51 smithi032 podman[334207]: 2024-09-21 10:35:51.760344873 +0000 UTC m=+0.744216356 container died 3d5db8fbb94c45fb074b3c4aec78730314b9c151e1e347d96eae687fd3db39b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T10:35:52.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T10:35:52.550 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:35:52.550 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T10:35:52.550 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T10:35:52.550 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.1 2024-09-21T10:35:52.890 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:52 smithi032 systemd[1]: Stopping Ceph osd.1 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:35:53.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:52 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[120382]: 2024-09-21T10:35:52.887+0000 7f453c7f5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:35:53.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:52 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[120382]: 2024-09-21T10:35:52.887+0000 7f453c7f5640 -1 osd.1 528 *** Got signal Terminated *** 2024-09-21T10:35:53.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:52 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1[120382]: 2024-09-21T10:35:52.887+0000 7f453c7f5640 -1 osd.1 528 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:35:58.682 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:58 smithi032 podman[334311]: 2024-09-21 10:35:58.375378972 +0000 UTC m=+5.600242389 container died 7166dcc673030830b63b5acc78407fde5b8a8acd94a055568d67d8103df85832 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T10:35:58.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:58 smithi032 podman[334311]: 2024-09-21 10:35:58.871672626 +0000 UTC m=+6.096536042 container cleanup 7166dcc673030830b63b5acc78407fde5b8a8acd94a055568d67d8103df85832 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b) 2024-09-21T10:35:58.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:58 smithi032 bash[334311]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1 2024-09-21T10:35:59.457 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:59 smithi032 podman[334326]: 2024-09-21 10:35:59.106505369 +0000 UTC m=+0.716847440 container remove 7166dcc673030830b63b5acc78407fde5b8a8acd94a055568d67d8103df85832 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T10:35:59.769 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:59 smithi032 podman[334419]: 2024-09-21 10:35:59.42059998 +0000 UTC m=+0.016509807 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:35:59.770 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:59 smithi032 podman[334419]: 2024-09-21 10:35:59.549098852 +0000 UTC m=+0.145008676 container create 563bd05f2e0875a4fbae2d7c611851379e1df696d206494c20aef381e8699d48 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:35:59.770 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:59 smithi032 podman[334419]: 2024-09-21 10:35:59.768077745 +0000 UTC m=+0.363987579 container init 563bd05f2e0875a4fbae2d7c611851379e1df696d206494c20aef381e8699d48 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T10:36:00.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:59 smithi032 podman[334419]: 2024-09-21 10:35:59.771454697 +0000 UTC m=+0.367364522 container start 563bd05f2e0875a4fbae2d7c611851379e1df696d206494c20aef381e8699d48 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T10:36:00.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:59 smithi032 podman[334419]: 2024-09-21 10:35:59.876431923 +0000 UTC m=+0.472341756 container attach 563bd05f2e0875a4fbae2d7c611851379e1df696d206494c20aef381e8699d48 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-62da54b, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T10:36:00.207 INFO:journalctl@ceph.osd.1.smithi032.stdout:Sep 21 10:35:59 smithi032 podman[334419]: 2024-09-21 10:35:59.966009112 +0000 UTC m=+0.561918936 container died 563bd05f2e0875a4fbae2d7c611851379e1df696d206494c20aef381e8699d48 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-21T10:36:00.750 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.1.service' 2024-09-21T10:36:00.796 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:36:00.797 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T10:36:00.797 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T10:36:00.797 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.2 2024-09-21T10:36:01.112 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:00 smithi032 systemd[1]: Stopping Ceph osd.2 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:36:01.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[124768]: 2024-09-21T10:36:01.109+0000 7f897233f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:36:01.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[124768]: 2024-09-21T10:36:01.109+0000 7f897233f640 -1 osd.2 528 *** Got signal Terminated *** 2024-09-21T10:36:01.458 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:01 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2[124768]: 2024-09-21T10:36:01.109+0000 7f897233f640 -1 osd.2 528 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:36:06.845 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:06 smithi032 podman[334522]: 2024-09-21 10:36:06.573199432 +0000 UTC m=+5.577555698 container died 84d2b69ff1bafc5ca35c22182b9e7244a9dcbd954f47da051c4250c68b1a4505 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:36:07.207 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:07 smithi032 podman[334522]: 2024-09-21 10:36:07.034906844 +0000 UTC m=+6.039263151 container cleanup 84d2b69ff1bafc5ca35c22182b9e7244a9dcbd954f47da051c4250c68b1a4505 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-62da54b, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T10:36:07.207 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:07 smithi032 bash[334522]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2 2024-09-21T10:36:07.707 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:07 smithi032 podman[334550]: 2024-09-21 10:36:07.353272064 +0000 UTC m=+0.776009210 container remove 84d2b69ff1bafc5ca35c22182b9e7244a9dcbd954f47da051c4250c68b1a4505 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T10:36:07.983 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:07 smithi032 podman[334639]: 2024-09-21 10:36:07.615070202 +0000 UTC m=+0.016611456 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:36:07.983 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:07 smithi032 podman[334639]: 2024-09-21 10:36:07.732767867 +0000 UTC m=+0.134309108 container create c416621b26bb4d1cfb9f24b488594050de409a51ca3088c4d3a5502fe7fa870a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:36:07.983 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:07 smithi032 podman[334639]: 2024-09-21 10:36:07.981864349 +0000 UTC m=+0.383405600 container init c416621b26bb4d1cfb9f24b488594050de409a51ca3088c4d3a5502fe7fa870a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T10:36:08.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:07 smithi032 podman[334639]: 2024-09-21 10:36:07.985475797 +0000 UTC m=+0.387017039 container start c416621b26bb4d1cfb9f24b488594050de409a51ca3088c4d3a5502fe7fa870a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b) 2024-09-21T10:36:08.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:08 smithi032 podman[334639]: 2024-09-21 10:36:08.065032563 +0000 UTC m=+0.466573813 container attach c416621b26bb4d1cfb9f24b488594050de409a51ca3088c4d3a5502fe7fa870a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T10:36:08.457 INFO:journalctl@ceph.osd.2.smithi032.stdout:Sep 21 10:36:08 smithi032 podman[334639]: 2024-09-21 10:36:08.18411628 +0000 UTC m=+0.585657527 container died c416621b26bb4d1cfb9f24b488594050de409a51ca3088c4d3a5502fe7fa870a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:36:09.086 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.2.service' 2024-09-21T10:36:09.129 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:36:09.130 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T10:36:09.130 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T10:36:09.130 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.3 2024-09-21T10:36:09.457 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:09 smithi032 systemd[1]: Stopping Ceph osd.3 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:36:09.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[129137]: 2024-09-21T10:36:09.463+0000 7ff68e513640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:36:09.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[129137]: 2024-09-21T10:36:09.463+0000 7ff68e513640 -1 osd.3 528 *** Got signal Terminated *** 2024-09-21T10:36:09.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:09 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[129137]: 2024-09-21T10:36:09.463+0000 7ff68e513640 -1 osd.3 528 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:36:11.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:10 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:10.727+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:12.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:11.685+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:12.035 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:11.613+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:12.036 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:11 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:11.593+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:12.844 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:12 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:12.708+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:12.845 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:12 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:12.581+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:12.845 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:12 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:12.568+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:13.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:12 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:12.842+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:13.839 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:13 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:13.663+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:13.839 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:13 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:13.543+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:13.839 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:13 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:13.542+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:14.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:13 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3[129137]: 2024-09-21T10:36:13.840+0000 7ff686b20640 -1 osd.3 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:47.440879+0000 front 2024-09-21T10:35:47.440933+0000 (oldest deadline 2024-09-21T10:36:13.241499+0000) 2024-09-21T10:36:14.286 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:13 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:13.836+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:14.785 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:14 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:14.536+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:14.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:14 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:14.498+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:14.785 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:14 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:14.644+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:15.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:14 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:14.849+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:15.408 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:15 smithi032 podman[334743]: 2024-09-21 10:36:15.123853588 +0000 UTC m=+5.760767909 container died e5f8912f154b3c1b70f73fb4ca1c8173e1d68acd8ec845298299d6742261015b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:36:15.706 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:15 smithi032 podman[334743]: 2024-09-21 10:36:15.596712613 +0000 UTC m=+6.233626889 container cleanup e5f8912f154b3c1b70f73fb4ca1c8173e1d68acd8ec845298299d6742261015b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T10:36:15.707 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:15 smithi032 bash[334743]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3 2024-09-21T10:36:15.900 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:15 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:15.575+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:15.900 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:15 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:15.537+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:15.901 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:15 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:15.672+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:16.207 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:15 smithi032 podman[334758]: 2024-09-21 10:36:15.8149509 +0000 UTC m=+0.683049344 container remove e5f8912f154b3c1b70f73fb4ca1c8173e1d68acd8ec845298299d6742261015b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-21T10:36:16.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:15 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:15.897+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:16.477 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:16 smithi032 podman[334848]: 2024-09-21 10:36:16.119448855 +0000 UTC m=+0.016295819 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:36:16.478 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:16 smithi032 podman[334848]: 2024-09-21 10:36:16.249011968 +0000 UTC m=+0.145858945 container create 63f2d075a38ebb18ef87433cd40f0c368bedf9b0d40c44c7f0dab1d6188adaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T10:36:16.478 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:16 smithi032 podman[334848]: 2024-09-21 10:36:16.476326556 +0000 UTC m=+0.373173510 container init 63f2d075a38ebb18ef87433cd40f0c368bedf9b0d40c44c7f0dab1d6188adaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T10:36:16.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:16 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:16.486+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:16.785 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:16 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:16.571+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:16.785 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:16 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:16.700+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:16.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:16 smithi032 podman[334848]: 2024-09-21 10:36:16.479761194 +0000 UTC m=+0.376608130 container start 63f2d075a38ebb18ef87433cd40f0c368bedf9b0d40c44c7f0dab1d6188adaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T10:36:16.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:16 smithi032 podman[334848]: 2024-09-21 10:36:16.576214147 +0000 UTC m=+0.473061093 container attach 63f2d075a38ebb18ef87433cd40f0c368bedf9b0d40c44c7f0dab1d6188adaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T10:36:16.957 INFO:journalctl@ceph.osd.3.smithi032.stdout:Sep 21 10:36:16 smithi032 podman[334848]: 2024-09-21 10:36:16.675457621 +0000 UTC m=+0.572304563 container died 63f2d075a38ebb18ef87433cd40f0c368bedf9b0d40c44c7f0dab1d6188adaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:36:17.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:16 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:16.940+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:17.347 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T10:36:17.389 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:36:17.389 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T10:36:17.390 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T10:36:17.390 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.4 2024-09-21T10:36:17.706 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:17 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:17.501+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:17.706 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:17 smithi045 systemd[1]: Stopping Ceph osd.4 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:36:17.707 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:17 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:17.527+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:17.707 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:17 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:17.703+0000 7ff1864a6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:36:17.707 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:17 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:17.703+0000 7ff1864a6640 -1 osd.4 528 *** Got signal Terminated *** 2024-09-21T10:36:18.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:17 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:17.931+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:18.035 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:17 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:17.703+0000 7ff1864a6640 -1 osd.4 528 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:36:18.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:17 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:17.726+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:18.785 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:18.488+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:18.785 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:18.730+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:18.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:18.524+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:19.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:18 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:18.901+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:19.776 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:19 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:19.446+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:19.777 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:19 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:19.446+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:54.665899+0000 front 2024-09-21T10:35:54.666041+0000 (oldest deadline 2024-09-21T10:36:18.765859+0000) 2024-09-21T10:36:19.777 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:19 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:19.495+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:20.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:19 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:19.897+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:20.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:19 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:19.774+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:20.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:20 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:20.456+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:20.785 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:20 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:20.496+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:20.785 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:20 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:20.496+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:54.665899+0000 front 2024-09-21T10:35:54.666041+0000 (oldest deadline 2024-09-21T10:36:18.765859+0000) 2024-09-21T10:36:21.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:20 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:20.875+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:21.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:20 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:20.790+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:21.773 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:21.429+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:21.774 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:21.429+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:21.774 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:21.520+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:21.774 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:21.520+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:54.665899+0000 front 2024-09-21T10:35:54.666041+0000 (oldest deadline 2024-09-21T10:36:18.765859+0000) 2024-09-21T10:36:22.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:21.771+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:22.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:21.771+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:22.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:21.867+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:22.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:21 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:21.867+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:22.731 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:22.564+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:49.465133+0000 front 2024-09-21T10:35:49.465191+0000 (oldest deadline 2024-09-21T10:36:11.164932+0000) 2024-09-21T10:36:22.731 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4[37896]: 2024-09-21T10:36:22.564+0000 7ff17eab2640 -1 osd.4 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:54.665899+0000 front 2024-09-21T10:35:54.666041+0000 (oldest deadline 2024-09-21T10:36:18.765859+0000) 2024-09-21T10:36:22.731 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:22.437+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:22.731 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:22.437+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:23.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:22.877+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:23.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:22.877+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:23.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:22.727+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:23.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:22 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:22.727+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:23.499 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:23 smithi045 podman[239287]: 2024-09-21 10:36:23.211712122 +0000 UTC m=+5.614129351 container died 81cba12364acd81061972df8510c94ad9b616767e868e4b81be9cf6c73e7c1ef (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=reef-6a1bdf8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T10:36:23.500 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:23 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:23.450+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:23.500 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:23 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:23.450+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:23.785 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:23 smithi045 podman[239287]: 2024-09-21 10:36:23.687118435 +0000 UTC m=+6.089535668 container cleanup 81cba12364acd81061972df8510c94ad9b616767e868e4b81be9cf6c73e7c1ef (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8) 2024-09-21T10:36:23.785 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:23 smithi045 bash[239287]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4 2024-09-21T10:36:23.785 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:23 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:23.708+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:23.785 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:23 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:23.708+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:24.285 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:24 smithi045 podman[239303]: 2024-09-21 10:36:24.113762576 +0000 UTC m=+0.896742764 container remove 81cba12364acd81061972df8510c94ad9b616767e868e4b81be9cf6c73e7c1ef (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T10:36:24.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:23 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:23.891+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:24.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:23 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:23.891+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:24.735 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:24.473+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:24.735 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:24.473+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:24.735 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:24 smithi045 podman[239392]: 2024-09-21 10:36:24.4647637 +0000 UTC m=+0.112062747 container create 2e960a98484097f063a70a766d3c97c1ead920ccb29d2fbd7da41c5acf82bd3f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T10:36:24.735 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:24 smithi045 podman[239392]: 2024-09-21 10:36:24.369041619 +0000 UTC m=+0.016340635 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:36:24.736 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:24 smithi045 podman[239392]: 2024-09-21 10:36:24.700573659 +0000 UTC m=+0.347872676 container init 2e960a98484097f063a70a766d3c97c1ead920ccb29d2fbd7da41c5acf82bd3f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:36:24.736 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:24 smithi045 podman[239392]: 2024-09-21 10:36:24.703873489 +0000 UTC m=+0.351172485 container start 2e960a98484097f063a70a766d3c97c1ead920ccb29d2fbd7da41c5acf82bd3f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T10:36:25.035 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:24 smithi045 podman[239392]: 2024-09-21 10:36:24.809100028 +0000 UTC m=+0.456399046 container attach 2e960a98484097f063a70a766d3c97c1ead920ccb29d2fbd7da41c5acf82bd3f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux ) 2024-09-21T10:36:25.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:24.863+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:25.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:24.863+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:25.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:24.732+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:25.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:24 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:24.732+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:25.328 INFO:journalctl@ceph.osd.4.smithi045.stdout:Sep 21 10:36:25 smithi045 podman[239392]: 2024-09-21 10:36:25.052799096 +0000 UTC m=+0.700098104 container died 2e960a98484097f063a70a766d3c97c1ead920ccb29d2fbd7da41c5acf82bd3f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:36:25.639 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:25.516+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:25.639 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:25.516+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:25.655 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.4.service' 2024-09-21T10:36:25.696 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:36:25.696 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T10:36:25.696 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T10:36:25.696 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.5 2024-09-21T10:36:25.891 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:25 smithi045 systemd[1]: Stopping Ceph osd.5 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:36:25.891 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:25.780+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:25.891 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:25.780+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:26.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:25.888+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:26.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:25 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:25.888+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:26.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:26.048+0000 7f2357aeb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:36:26.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:26.048+0000 7f2357aeb640 -1 osd.5 528 *** Got signal Terminated *** 2024-09-21T10:36:26.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:26.048+0000 7f2357aeb640 -1 osd.5 528 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:36:26.744 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:26.536+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:26.745 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:26.536+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:27.034 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:26.870+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:27.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:26.870+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:27.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:26.741+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:27.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:26 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:26.741+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:27.786 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:27 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:27.750+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:27.786 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:27 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:27.750+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:27.786 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:27 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:27.501+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:27.786 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:27 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:27.501+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:28.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:27 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:27.872+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:28.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:27 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:27.872+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:28.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:28.485+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:28.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:28.485+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:29.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:28.894+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:29.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:28.894+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:29.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:28.798+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:29.286 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:28.798+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:29.286 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:28 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:28.798+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:29.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:29.513+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:29.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:29.513+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:29.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:29.513+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:29.785 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:29.763+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:29.786 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:29.763+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:29.786 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:29.763+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:30.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:29.906+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:30.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:29 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:29.906+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:30.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:30.516+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:30.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:30.516+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:30.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:30.516+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:30.785 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:30.714+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:30.785 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:30.714+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:30.786 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:30.714+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:31.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:30.893+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.645385+0000 front 2024-09-21T10:35:46.645630+0000 (oldest deadline 2024-09-21T10:36:11.945270+0000) 2024-09-21T10:36:31.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:30.893+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:57.246133+0000 front 2024-09-21T10:35:57.246227+0000 (oldest deadline 2024-09-21T10:36:21.346047+0000) 2024-09-21T10:36:31.285 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:30 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5[169770]: 2024-09-21T10:36:30.893+0000 7f23500f7640 -1 osd.5 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:04.747649+0000 front 2024-09-21T10:36:04.747722+0000 (oldest deadline 2024-09-21T10:36:30.647436+0000) 2024-09-21T10:36:31.727 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:31 smithi045 podman[239505]: 2024-09-21 10:36:31.47485138 +0000 UTC m=+5.532325104 container died e525a7eff74a3049d475040435a339a72e12935258634244ea6d61120359e5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-21T10:36:31.727 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:31.514+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:31.727 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:31.514+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:31.727 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:31.514+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:32.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:31 smithi045 podman[239505]: 2024-09-21 10:36:31.865902185 +0000 UTC m=+5.923375899 container cleanup e525a7eff74a3049d475040435a339a72e12935258634244ea6d61120359e5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:36:32.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:31 smithi045 bash[239505]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5 2024-09-21T10:36:32.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:31.724+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:32.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:31.724+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:32.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:31 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:31.724+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:32.478 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:32 smithi045 podman[239523]: 2024-09-21 10:36:32.15917761 +0000 UTC m=+0.673571565 container remove e525a7eff74a3049d475040435a339a72e12935258634244ea6d61120359e5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-21T10:36:32.747 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:32.486+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:32.748 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:32.486+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:32.748 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:32.486+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:32.748 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:32 smithi045 podman[239610]: 2024-09-21 10:36:32.476625408 +0000 UTC m=+0.110212026 container create 5c4a98431ca5472542fd9ce9e9416ffa36f143cac1fce896d3f9d149973f27de (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-21T10:36:32.749 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:32 smithi045 podman[239610]: 2024-09-21 10:36:32.382757762 +0000 UTC m=+0.016344397 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:36:32.749 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:32 smithi045 podman[239610]: 2024-09-21 10:36:32.721075006 +0000 UTC m=+0.354661630 container init 5c4a98431ca5472542fd9ce9e9416ffa36f143cac1fce896d3f9d149973f27de (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:36:32.749 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:32 smithi045 podman[239610]: 2024-09-21 10:36:32.725025425 +0000 UTC m=+0.358612044 container start 5c4a98431ca5472542fd9ce9e9416ffa36f143cac1fce896d3f9d149973f27de (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=reef-6a1bdf8, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T10:36:33.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:32 smithi045 podman[239610]: 2024-09-21 10:36:32.82939488 +0000 UTC m=+0.462981505 container attach 5c4a98431ca5472542fd9ce9e9416ffa36f143cac1fce896d3f9d149973f27de (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=reef-6a1bdf8) 2024-09-21T10:36:33.035 INFO:journalctl@ceph.osd.5.smithi045.stdout:Sep 21 10:36:32 smithi045 podman[239610]: 2024-09-21 10:36:32.924704034 +0000 UTC m=+0.558290671 container died 5c4a98431ca5472542fd9ce9e9416ffa36f143cac1fce896d3f9d149973f27de (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T10:36:33.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:32.744+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:33.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:32.744+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:33.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:32 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:32.744+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:33.532 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.5.service' 2024-09-21T10:36:33.578 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:36:33.578 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T10:36:33.578 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T10:36:33.578 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.6 2024-09-21T10:36:33.754 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:33.500+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:33.754 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:33.500+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:33.754 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:33.500+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:33.755 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:33 smithi045 systemd[1]: Stopping Ceph osd.6 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:36:34.035 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:33.877+0000 7fb8d25b7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:36:34.035 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:33.877+0000 7fb8d25b7640 -1 osd.6 528 *** Got signal Terminated *** 2024-09-21T10:36:34.035 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:33.877+0000 7fb8d25b7640 -1 osd.6 528 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:36:34.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:33.751+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:34.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:33.751+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:34.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:33 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:33.751+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:34.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:34 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:34.531+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:34.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:34 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:34.531+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:34.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:34 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:34.531+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:35.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:34 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:34.792+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:35.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:34 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:34.792+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:35.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:34 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:34.792+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:35.819 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:35 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:35.547+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:35.819 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:35 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:35.547+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:35.819 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:35 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:35.547+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:36.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:35 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:35.815+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:36.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:35 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:35.816+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:36.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:35 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:35.816+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:36.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:36 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:36.581+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:36.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:36 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:36.581+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:36.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:36 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:36.581+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:37.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:36 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:36.865+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:37.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:36 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:36.865+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:37.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:36 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:36.865+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:37.893 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:37 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:37.551+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:37.893 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:37 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:37.551+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:37.893 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:37 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:37.551+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:37.893 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:37 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:37.551+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:12.826888+0000 front 2024-09-21T10:36:12.826946+0000 (oldest deadline 2024-09-21T10:36:37.526896+0000) 2024-09-21T10:36:38.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:37 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:37.891+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:38.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:37 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:37.891+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:38.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:37 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:37.891+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:38.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:38.527+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:44.823972+0000 front 2024-09-21T10:35:44.824005+0000 (oldest deadline 2024-09-21T10:36:10.723824+0000) 2024-09-21T10:36:38.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:38.527+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:55.324857+0000 front 2024-09-21T10:35:55.324982+0000 (oldest deadline 2024-09-21T10:36:20.624906+0000) 2024-09-21T10:36:38.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:38.527+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:05.826185+0000 front 2024-09-21T10:36:05.826399+0000 (oldest deadline 2024-09-21T10:36:28.726202+0000) 2024-09-21T10:36:38.785 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6[49201]: 2024-09-21T10:36:38.527+0000 7fb8cebcb640 -1 osd.6 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:12.826888+0000 front 2024-09-21T10:36:12.826946+0000 (oldest deadline 2024-09-21T10:36:37.526896+0000) 2024-09-21T10:36:39.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:38.897+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:39.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:38.897+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:39.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:38.897+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:39.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:38 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:38.897+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:13.680414+0000 front 2024-09-21T10:36:13.680518+0000 (oldest deadline 2024-09-21T10:36:38.380446+0000) 2024-09-21T10:36:39.781 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:39 smithi045 podman[239712]: 2024-09-21 10:36:39.378468823 +0000 UTC m=+5.602795337 container died ad56e76fa9f233869bd38bdfd37cec13789f15b94f6304c1cff042506e4a170e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731) 2024-09-21T10:36:40.035 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:39 smithi045 podman[239712]: 2024-09-21 10:36:39.894201507 +0000 UTC m=+6.118528018 container cleanup ad56e76fa9f233869bd38bdfd37cec13789f15b94f6304c1cff042506e4a170e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T10:36:40.035 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:39 smithi045 bash[239712]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6 2024-09-21T10:36:40.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:39 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:39.885+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:40.035 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:39 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:39.885+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:40.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:39 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:39.885+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:40.036 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:39 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:39.885+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:13.680414+0000 front 2024-09-21T10:36:13.680518+0000 (oldest deadline 2024-09-21T10:36:38.380446+0000) 2024-09-21T10:36:40.535 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:40 smithi045 podman[239727]: 2024-09-21 10:36:40.162582719 +0000 UTC m=+0.779376308 container remove ad56e76fa9f233869bd38bdfd37cec13789f15b94f6304c1cff042506e4a170e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T10:36:40.875 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:40 smithi045 podman[239813]: 2024-09-21 10:36:40.473220143 +0000 UTC m=+0.016912527 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:36:40.875 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:40 smithi045 podman[239813]: 2024-09-21 10:36:40.580141423 +0000 UTC m=+0.123833833 container create 2fd6d8573a6932c1fa8a18365de882198e31e2f4586f80ad8ad764b12867fc32 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T10:36:40.876 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:40 smithi045 podman[239813]: 2024-09-21 10:36:40.790784397 +0000 UTC m=+0.334476773 container init 2fd6d8573a6932c1fa8a18365de882198e31e2f4586f80ad8ad764b12867fc32 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=reef-6a1bdf8, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T10:36:40.876 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:40 smithi045 podman[239813]: 2024-09-21 10:36:40.794126721 +0000 UTC m=+0.337819088 container start 2fd6d8573a6932c1fa8a18365de882198e31e2f4586f80ad8ad764b12867fc32 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=reef-6a1bdf8, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:36:40.876 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:40 smithi045 podman[239813]: 2024-09-21 10:36:40.87398868 +0000 UTC m=+0.417681061 container attach 2fd6d8573a6932c1fa8a18365de882198e31e2f4586f80ad8ad764b12867fc32 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:36:41.279 INFO:journalctl@ceph.osd.6.smithi045.stdout:Sep 21 10:36:40 smithi045 podman[239813]: 2024-09-21 10:36:40.983868111 +0000 UTC m=+0.527560481 container died 2fd6d8573a6932c1fa8a18365de882198e31e2f4586f80ad8ad764b12867fc32 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-21T10:36:41.279 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:40.882+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:41.279 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:40.882+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:41.280 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:40.882+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:41.280 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:40 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:40.882+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:13.680414+0000 front 2024-09-21T10:36:13.680518+0000 (oldest deadline 2024-09-21T10:36:38.380446+0000) 2024-09-21T10:36:41.600 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.6.service' 2024-09-21T10:36:41.644 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:36:41.644 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T10:36:41.644 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T10:36:41.644 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.7 2024-09-21T10:36:41.785 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:41 smithi045 systemd[1]: Stopping Ceph osd.7 for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:36:41.785 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:41.532Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T10:36:41.785 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:41.533Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T10:36:41.785 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:41.537Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T10:36:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:41.537Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T10:36:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:41.537Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T10:36:41.786 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:41.538Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T10:36:42.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:41.929+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:42.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:41.929+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:42.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:41.929+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:42.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:41.929+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:13.680414+0000 front 2024-09-21T10:36:13.680518+0000 (oldest deadline 2024-09-21T10:36:38.380446+0000) 2024-09-21T10:36:42.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:41.972+0000 7f7b68808640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:36:42.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:41.972+0000 7f7b68808640 -1 osd.7 528 *** Got signal Terminated *** 2024-09-21T10:36:42.286 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:41 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:41.972+0000 7f7b68808640 -1 osd.7 528 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:36:42.974 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:42 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:42.570Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.32:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.32:8765: connect: connection refused" 2024-09-21T10:36:43.284 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:42 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:42.972+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:43.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:42 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:42.972+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:43.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:42 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:42.972+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:43.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:42 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:42.972+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:13.680414+0000 front 2024-09-21T10:36:13.680518+0000 (oldest deadline 2024-09-21T10:36:38.380446+0000) 2024-09-21T10:36:44.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:43 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:43.994+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:44.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:43 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:43.994+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:44.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:43 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:43.994+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:44.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:43 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:43.994+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:13.680414+0000 front 2024-09-21T10:36:13.680518+0000 (oldest deadline 2024-09-21T10:36:38.380446+0000) 2024-09-21T10:36:45.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:45 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:45.021+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:45.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:45 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:45.021+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:45.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:45 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:45.021+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:45.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:45 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:45.021+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:13.680414+0000 front 2024-09-21T10:36:13.680518+0000 (oldest deadline 2024-09-21T10:36:38.380446+0000) 2024-09-21T10:36:45.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:45 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:45.021+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.45:6804 osd.4 since back 2024-09-21T10:36:19.481279+0000 front 2024-09-21T10:36:19.481429+0000 (oldest deadline 2024-09-21T10:36:44.781200+0000) 2024-09-21T10:36:46.535 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:46 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:46.037+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-09-21T10:35:46.375963+0000 front 2024-09-21T10:35:46.376128+0000 (oldest deadline 2024-09-21T10:36:10.475974+0000) 2024-09-21T10:36:46.535 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:46 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:46.037+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-09-21T10:35:56.877806+0000 front 2024-09-21T10:35:56.877868+0000 (oldest deadline 2024-09-21T10:36:20.977655+0000) 2024-09-21T10:36:46.535 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:46 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:46.037+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-09-21T10:36:03.779234+0000 front 2024-09-21T10:36:03.779191+0000 (oldest deadline 2024-09-21T10:36:28.479093+0000) 2024-09-21T10:36:46.535 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:46 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:46.037+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-09-21T10:36:13.680414+0000 front 2024-09-21T10:36:13.680518+0000 (oldest deadline 2024-09-21T10:36:38.380446+0000) 2024-09-21T10:36:46.535 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:46 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7[54732]: 2024-09-21T10:36:46.037+0000 7f7b64e1c640 -1 osd.7 528 heartbeat_check: no reply from 172.21.15.45:6804 osd.4 since back 2024-09-21T10:36:19.481279+0000 front 2024-09-21T10:36:19.481429+0000 (oldest deadline 2024-09-21T10:36:44.781200+0000) 2024-09-21T10:36:47.891 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:47 smithi045 podman[239915]: 2024-09-21 10:36:47.619221739 +0000 UTC m=+5.748513514 container died 8a3d61224a1bb0a0e6f388914bb86a4ee1ed4228232c83595aab955a982d47e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T10:36:48.274 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:47 smithi045 podman[239915]: 2024-09-21 10:36:47.988675819 +0000 UTC m=+6.117967579 container cleanup 8a3d61224a1bb0a0e6f388914bb86a4ee1ed4228232c83595aab955a982d47e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=reef-6a1bdf8, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:36:48.275 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:47 smithi045 bash[239915]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7 2024-09-21T10:36:48.535 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:48 smithi045 podman[239930]: 2024-09-21 10:36:48.273235511 +0000 UTC m=+0.653154963 container remove 8a3d61224a1bb0a0e6f388914bb86a4ee1ed4228232c83595aab955a982d47e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=reef-6a1bdf8) 2024-09-21T10:36:48.853 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:48 smithi045 podman[240020]: 2024-09-21 10:36:48.491980545 +0000 UTC m=+0.016747344 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:36:48.853 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:48 smithi045 podman[240020]: 2024-09-21 10:36:48.590595194 +0000 UTC m=+0.115361992 container create 22e4e9fa5fc23625eab5baba00e685c16bec263ba86d3f56ddcab4be67706a11 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:36:48.853 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:48 smithi045 podman[240020]: 2024-09-21 10:36:48.851385117 +0000 UTC m=+0.376151924 container init 22e4e9fa5fc23625eab5baba00e685c16bec263ba86d3f56ddcab4be67706a11 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0) 2024-09-21T10:36:49.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:48 smithi045 podman[240020]: 2024-09-21 10:36:48.855014321 +0000 UTC m=+0.379781120 container start 22e4e9fa5fc23625eab5baba00e685c16bec263ba86d3f56ddcab4be67706a11 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:36:49.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:48 smithi045 podman[240020]: 2024-09-21 10:36:48.943090291 +0000 UTC m=+0.467857107 container attach 22e4e9fa5fc23625eab5baba00e685c16bec263ba86d3f56ddcab4be67706a11 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T10:36:49.285 INFO:journalctl@ceph.osd.7.smithi045.stdout:Sep 21 10:36:49 smithi045 podman[240020]: 2024-09-21 10:36:49.050863014 +0000 UTC m=+0.575629814 container died 22e4e9fa5fc23625eab5baba00e685c16bec263ba86d3f56ddcab4be67706a11 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T10:36:49.703 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@osd.7.service' 2024-09-21T10:36:49.743 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:36:49.744 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T10:36:49.744 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-21T10:36:49.744 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a 2024-09-21T10:36:50.035 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:49 smithi045 systemd[1]: Stopping Ceph prometheus.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:36:50.386 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.083Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T10:36:50.386 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.083Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-21T10:36:50.386 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.083Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-21T10:36:50.386 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.083Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T10:36:50.386 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.083Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-21T10:36:50.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.083Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-21T10:36:50.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.084Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T10:36:50.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.084Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-21T10:36:50.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.084Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-21T10:36:50.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.085Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T10:36:50.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.085Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-21T10:36:50.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a[83817]: ts=2024-09-21T10:36:50.085Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-21T10:36:50.387 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 podman[240123]: 2024-09-21 10:36:50.093990861 +0000 UTC m=+0.126962757 container died 2d9f1cee2e82c4dce50997db04bdf78f6c43d60b0a52ff8c69a4db1fbb8954a3 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T10:36:50.760 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 podman[240123]: 2024-09-21 10:36:50.483313317 +0000 UTC m=+0.516285205 container cleanup 2d9f1cee2e82c4dce50997db04bdf78f6c43d60b0a52ff8c69a4db1fbb8954a3 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T10:36:50.761 INFO:journalctl@ceph.prometheus.a.smithi045.stdout:Sep 21 10:36:50 smithi045 bash[240123]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T10:36:50.889 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@prometheus.a.service' 2024-09-21T10:36:50.923 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:36:50.924 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-21T10:36:50.924 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:36:51.084 INFO:teuthology.orchestra.run.smithi032.stdout:Deleting cluster with fsid: e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T10:36:52.621 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:52 smithi032 systemd[1]: Stopping Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:36:52.957 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:52 smithi032 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a[96743]: ts=2024-09-21T10:36:52.805Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T10:36:52.957 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:52 smithi032 podman[335249]: 2024-09-21 10:36:52.84665149 +0000 UTC m=+0.160603121 container died d6fa4f2dece853e9a0d502d86f167b2dac9001bedad89290ee0fe27abee67b45 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:36:53.456 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:53 smithi032 podman[335249]: 2024-09-21 10:36:53.339293144 +0000 UTC m=+0.653244773 container cleanup d6fa4f2dece853e9a0d502d86f167b2dac9001bedad89290ee0fe27abee67b45 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:36:53.457 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:53 smithi032 bash[335249]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T10:36:53.904 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:53 smithi032 podman[335264]: 2024-09-21 10:36:53.615753124 +0000 UTC m=+0.760678300 container remove d6fa4f2dece853e9a0d502d86f167b2dac9001bedad89290ee0fe27abee67b45 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:36:53.904 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:53 smithi032 podman[335264]: 2024-09-21 10:36:53.699878908 +0000 UTC m=+0.844804041 volume remove 70e01a1d853493fb928a16458a23d9b6741a7bd564a635ee1e3f5f32f0b7ed0a 2024-09-21T10:36:54.207 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 10:36:54 smithi032 systemd[1]: Stopping Ceph node-exporter.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:36:54.207 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:53 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T10:36:54.207 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:53 smithi032 systemd[1]: Stopped Ceph alertmanager.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:36:54.207 INFO:journalctl@ceph.alertmanager.a.smithi032.stdout:Sep 21 10:36:53 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 3.122s CPU time. 2024-09-21T10:36:54.654 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 10:36:54 smithi032 podman[335372]: 2024-09-21 10:36:54.396638652 +0000 UTC m=+0.121319018 container died e54d1ecd8cb39af5350faa58eec1ea4053cc2643d4dec51523579ba1df8feb6f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:36:54.956 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 10:36:54 smithi032 podman[335372]: 2024-09-21 10:36:54.752697375 +0000 UTC m=+0.477377741 container cleanup e54d1ecd8cb39af5350faa58eec1ea4053cc2643d4dec51523579ba1df8feb6f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:36:54.957 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 10:36:54 smithi032 bash[335372]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T10:36:55.335 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 10:36:55 smithi032 podman[335386]: 2024-09-21 10:36:55.020426535 +0000 UTC m=+0.621897256 container remove e54d1ecd8cb39af5350faa58eec1ea4053cc2643d4dec51523579ba1df8feb6f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:36:55.335 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 10:36:55 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:36:55.335 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 10:36:55 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T10:36:55.335 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 10:36:55 smithi032 systemd[1]: Stopped Ceph node-exporter.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:36:55.335 INFO:journalctl@ceph.node-exporter.a.smithi032.stdout:Sep 21 10:36:55 smithi032 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 7.624s CPU time. 2024-09-21T10:37:04.837 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:37:04.996 INFO:teuthology.orchestra.run.smithi045.stdout:Deleting cluster with fsid: e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T10:37:06.733 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:06 smithi045 systemd[1]: Stopping Ceph grafana.a for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:37:07.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:06 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=server t=2024-09-21T10:37:06.754888256Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-21T10:37:07.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:06 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=tracing t=2024-09-21T10:37:06.755200671Z level=info msg="Closing tracing" 2024-09-21T10:37:07.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:06 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=ticker t=2024-09-21T10:37:06.755211955Z level=info msg=stopped last_tick=2024-09-21T10:37:00Z 2024-09-21T10:37:07.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:06 smithi045 ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a[88232]: logger=grafana-apiserver t=2024-09-21T10:37:06.755257653Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-21T10:37:07.035 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:06 smithi045 podman[240542]: 2024-09-21 10:37:06.775301265 +0000 UTC m=+0.134695286 container died 0ad08e43d776ae6e8699fb6d3d73d3840e557d0acc14464f2c8068ad27ad1168 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:37:07.423 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:07 smithi045 podman[240542]: 2024-09-21 10:37:07.162351915 +0000 UTC m=+0.521745978 container cleanup 0ad08e43d776ae6e8699fb6d3d73d3840e557d0acc14464f2c8068ad27ad1168 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:37:07.423 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:07 smithi045 bash[240542]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a 2024-09-21T10:37:07.742 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:07 smithi045 podman[240556]: 2024-09-21 10:37:07.421669178 +0000 UTC m=+0.643136823 container remove 0ad08e43d776ae6e8699fb6d3d73d3840e557d0acc14464f2c8068ad27ad1168 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:37:07.742 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:07 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@grafana.a.service: Deactivated successfully. 2024-09-21T10:37:07.742 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:07 smithi045 systemd[1]: Stopped Ceph grafana.a for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:37:07.742 INFO:journalctl@ceph.grafana.a.smithi045.stdout:Sep 21 10:37:07 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@grafana.a.service: Consumed 30.486s CPU time. 2024-09-21T10:37:08.029 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 10:37:07 smithi045 systemd[1]: Stopping Ceph node-exporter.b for e0bde4d4-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:37:08.284 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 10:37:08 smithi045 podman[240664]: 2024-09-21 10:37:08.027586767 +0000 UTC m=+0.142444693 container died 61f6c7aa590c5a5d99eeaa09c5df2c4065bc43be8da20c7759e537f23a0c4c7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:37:08.685 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 10:37:08 smithi045 podman[240664]: 2024-09-21 10:37:08.37452677 +0000 UTC m=+0.489384694 container cleanup 61f6c7aa590c5a5d99eeaa09c5df2c4065bc43be8da20c7759e537f23a0c4c7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:37:08.685 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 10:37:08 smithi045 bash[240664]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T10:37:09.011 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 10:37:08 smithi045 podman[240677]: 2024-09-21 10:37:08.684272034 +0000 UTC m=+0.659920822 container remove 61f6c7aa590c5a5d99eeaa09c5df2c4065bc43be8da20c7759e537f23a0c4c7c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:37:09.011 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 10:37:08 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:37:09.011 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 10:37:08 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T10:37:09.012 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 10:37:08 smithi045 systemd[1]: Stopped Ceph node-exporter.b for e0bde4d4-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:37:09.012 INFO:journalctl@ceph.node-exporter.b.smithi045.stdout:Sep 21 10:37:08 smithi045 systemd[1]: ceph-e0bde4d4-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 7.447s CPU time. 2024-09-21T10:37:12.365 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:37:12.399 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:37:12.428 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T10:37:12.430 DEBUG:teuthology.misc:Transferring archived files from smithi032:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913660/remote/smithi032/crash 2024-09-21T10:37:12.431 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:37:12.487 INFO:teuthology.orchestra.run.smithi032.stderr:tar: /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:37:12.487 INFO:teuthology.orchestra.run.smithi032.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:37:12.489 DEBUG:teuthology.misc:Transferring archived files from smithi045:/var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913660/remote/smithi045/crash 2024-09-21T10:37:12.490 DEBUG:teuthology.orchestra.run.smithi045:> sudo tar c -f - -C /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:37:12.528 INFO:teuthology.orchestra.run.smithi045.stderr:tar: /var/lib/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:37:12.528 INFO:teuthology.orchestra.run.smithi045.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:37:12.529 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T10:37:12.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-21T10:37:12.625 INFO:teuthology.orchestra.run.smithi032.stdout:2024-09-21T09:49:11.988563+0000 mon.a (mon.0) 695 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:37:12.628 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-21T10:37:12.628 INFO:tasks.cephadm:Compressing logs... 2024-09-21T10:37:12.628 DEBUG:teuthology.orchestra.run.smithi032:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:37:12.630 DEBUG:teuthology.orchestra.run.smithi045:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:37:12.673 INFO:teuthology.orchestra.run.smithi032.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-21T10:37:12.673 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.0.log 2024-09-21T10:37:12.673 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.1.log 2024-09-21T10:37:12.674 INFO:teuthology.orchestra.run.smithi045.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-21T10:37:12.675 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5 --verbose --/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.0.log: /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:37:12.675 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:37:12.675 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mon.c.log 2024-09-21T10:37:12.679 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:37:12.679 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mon.a.log 2024-09-21T10:37:12.679 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.2.log 2024-09-21T10:37:12.679 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mgr.y.log 2024-09-21T10:37:12.679 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:37:12.679 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mds.foo.smithi032.fdolcm.log 2024-09-21T10:37:12.679 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.3.log 2024-09-21T10:37:12.679 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mds.foo.smithi032.fdolcm.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:37:12.680 INFO:teuthology.orchestra.run.smithi032.stderr: 85.8% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mds.foo.smithi032.fdolcm.log.gz 2024-09-21T10:37:12.680 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.7.log 2024-09-21T10:37:12.681 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:37:12.681 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.4.log 2024-09-21T10:37:12.681 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.7.log: /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:37:12.681 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:37:12.681 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mds.foo.smithi045.jnfsfb.log 2024-09-21T10:37:12.681 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.audit.log: /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.6.log 2024-09-21T10:37:12.681 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mds.foo.smithi045.jnfsfb.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.5.log 2024-09-21T10:37:12.682 INFO:teuthology.orchestra.run.smithi045.stderr: 84.0% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mds.foo.smithi045.jnfsfb.log.gz 2024-09-21T10:37:12.682 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:37:12.682 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mgr.x.log 2024-09-21T10:37:12.682 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mon.b.log 2024-09-21T10:37:12.682 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:37:12.686 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.8% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:37:12.961 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mon.b.log: /var/log/ceph/cephadm.log: 85.8% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:37:13.056 INFO:teuthology.orchestra.run.smithi032.stderr: 94.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:37:13.086 INFO:teuthology.orchestra.run.smithi045.stderr: 91.3% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:37:13.102 INFO:teuthology.orchestra.run.smithi045.stderr: 87.1% 90.5% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:37:13.102 INFO:teuthology.orchestra.run.smithi045.stderr: -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:37:13.102 INFO:teuthology.orchestra.run.smithi045.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:37:13.141 INFO:teuthology.orchestra.run.smithi045.stderr: 89.8% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mgr.x.log.gz 2024-09-21T10:37:13.413 INFO:teuthology.orchestra.run.smithi032.stderr: 94.3% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:37:13.912 INFO:teuthology.orchestra.run.smithi032.stderr: 92.0% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:37:13.927 INFO:teuthology.orchestra.run.smithi032.stderr: 91.2% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:37:21.003 INFO:teuthology.orchestra.run.smithi032.stderr: 89.7% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mgr.y.log.gz 2024-09-21T10:37:31.362 INFO:teuthology.orchestra.run.smithi032.stderr: 91.7% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mon.c.log.gz 2024-09-21T10:37:33.304 INFO:teuthology.orchestra.run.smithi045.stderr: 92.3% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mon.b.log.gz 2024-09-21T10:37:40.356 INFO:teuthology.orchestra.run.smithi032.stderr: 90.4% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-mon.a.log.gz 2024-09-21T10:39:42.129 INFO:teuthology.orchestra.run.smithi045.stderr: 93.6% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.7.log.gz 2024-09-21T10:43:15.147 INFO:teuthology.orchestra.run.smithi032.stderr: 93.1% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.1.log.gz 2024-09-21T10:48:40.614 INFO:teuthology.orchestra.run.smithi032.stderr: 93.6% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.0.log.gz 2024-09-21T10:49:57.930 INFO:teuthology.orchestra.run.smithi045.stderr: 93.0% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.4.log.gz 2024-09-21T10:51:29.662 INFO:teuthology.orchestra.run.smithi045.stderr: 93.2% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.6.log.gz 2024-09-21T10:51:46.431 INFO:teuthology.orchestra.run.smithi045.stderr: 92.9% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.5.log.gz 2024-09-21T10:51:46.433 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-21T10:51:46.433 INFO:teuthology.orchestra.run.smithi045.stderr:real 14m33.787s 2024-09-21T10:51:46.433 INFO:teuthology.orchestra.run.smithi045.stderr:user 6m42.237s 2024-09-21T10:51:46.433 INFO:teuthology.orchestra.run.smithi045.stderr:sys 0m18.557s 2024-09-21T10:52:10.804 INFO:teuthology.orchestra.run.smithi032.stderr: 93.5% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.2.log.gz 2024-09-21T10:53:12.037 INFO:teuthology.orchestra.run.smithi032.stderr: 92.8% -- replaced with /var/log/ceph/e0bde4d4-77fd-11ef-baf6-efdc52797490/ceph-osd.3.log.gz 2024-09-21T10:53:12.040 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-09-21T10:53:12.040 INFO:teuthology.orchestra.run.smithi032.stderr:real 15m59.394s 2024-09-21T10:53:12.040 INFO:teuthology.orchestra.run.smithi032.stderr:user 7m13.179s 2024-09-21T10:53:12.040 INFO:teuthology.orchestra.run.smithi032.stderr:sys 0m21.087s 2024-09-21T10:53:12.040 INFO:tasks.cephadm:Archiving logs... 2024-09-21T10:53:12.041 DEBUG:teuthology.misc:Transferring archived files from smithi032:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913660/remote/smithi032/log 2024-09-21T10:53:12.042 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:54:11.187 DEBUG:teuthology.misc:Transferring archived files from smithi045:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913660/remote/smithi045/log 2024-09-21T10:54:11.188 DEBUG:teuthology.orchestra.run.smithi045:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:55:05.711 INFO:tasks.cephadm:Removing cluster... 2024-09-21T10:55:05.712 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 --force 2024-09-21T10:55:05.889 INFO:teuthology.orchestra.run.smithi032.stdout:Deleting cluster with fsid: e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T10:55:06.666 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e0bde4d4-77fd-11ef-baf6-efdc52797490 --force 2024-09-21T10:55:06.867 INFO:teuthology.orchestra.run.smithi045.stdout:Deleting cluster with fsid: e0bde4d4-77fd-11ef-baf6-efdc52797490 2024-09-21T10:55:07.646 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T10:55:07.646 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:55:07.665 DEBUG:teuthology.orchestra.run.smithi045:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:55:07.687 INFO:tasks.cephadm:Teardown complete 2024-09-21T10:55:07.687 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-21T10:55:07.702 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-21T10:55:07.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T10:55:07.708 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T10:55:07.842 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T10:55:07.851 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T10:55:07.851 DEBUG:teuthology.orchestra.run.smithi032:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:55:07.854 DEBUG:teuthology.orchestra.run.smithi045:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:55:07.868 INFO:teuthology.orchestra.run.smithi032.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:55:07.869 INFO:teuthology.orchestra.run.smithi045.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:55:07.898 INFO:teuthology.orchestra.run.smithi045.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:55:07.898 INFO:teuthology.orchestra.run.smithi045.stdout:=============================================================================== 2024-09-21T10:55:07.898 INFO:teuthology.orchestra.run.smithi045.stdout:^* hv01.front.sepia.ceph.com 3 6 377 62 -185us[ -180us] +/- 51ms 2024-09-21T10:55:07.898 INFO:teuthology.orchestra.run.smithi045.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 58 +218us[ +218us] +/- 60ms 2024-09-21T10:55:07.898 INFO:teuthology.orchestra.run.smithi045.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 60 +4148ns[+4148ns] +/- 62ms 2024-09-21T10:55:07.899 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:55:07.909 INFO:teuthology.orchestra.run.smithi032.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:55:07.909 INFO:teuthology.orchestra.run.smithi032.stdout:=============================================================================== 2024-09-21T10:55:07.909 INFO:teuthology.orchestra.run.smithi032.stdout:^* hv01.front.sepia.ceph.com 3 6 377 60 -183us[ -178us] +/- 51ms 2024-09-21T10:55:07.909 INFO:teuthology.orchestra.run.smithi032.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 60 +225us[ +230us] +/- 60ms 2024-09-21T10:55:07.910 INFO:teuthology.orchestra.run.smithi032.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 57 -6052ns[-6052ns] +/- 62ms 2024-09-21T10:55:07.910 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:55:07.911 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T10:55:07.920 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T10:55:07.920 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T10:55:07.974 DEBUG:teuthology.orchestra.run.smithi032:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:55:08.100 DEBUG:teuthology.orchestra.run.smithi045:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:55:08.242 DEBUG:teuthology.orchestra.run.smithi032:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-21T10:55:08.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:08.274 DEBUG:teuthology.orchestra.run.smithi045:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-21T10:55:08.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:08.303 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T10:55:08.313 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T10:55:08.323 INFO:teuthology.task.internal:Duration was 4692.238032 seconds 2024-09-21T10:55:08.323 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T10:55:08.332 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T10:55:08.333 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:55:08.335 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:55:08.385 INFO:teuthology.orchestra.run.smithi032.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:55:08.425 INFO:teuthology.orchestra.run.smithi045.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:55:08.768 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T10:55:08.768 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi032.front.sepia.ceph.com 2024-09-21T10:55:08.769 DEBUG:teuthology.orchestra.run.smithi032:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:55:08.804 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi045.front.sepia.ceph.com 2024-09-21T10:55:08.804 DEBUG:teuthology.orchestra.run.smithi045:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:55:08.833 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T10:55:08.834 DEBUG:teuthology.orchestra.run.smithi032:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:55:08.847 DEBUG:teuthology.orchestra.run.smithi045:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:55:08.906 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T10:55:08.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:55:08.917 DEBUG:teuthology.orchestra.run.smithi045:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:55:10.593 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T10:55:10.603 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T10:55:10.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:55:10.651 DEBUG:teuthology.orchestra.run.smithi045:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:55:10.692 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T10:55:10.702 DEBUG:teuthology.orchestra.run.smithi032:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:55:10.705 DEBUG:teuthology.orchestra.run.smithi045:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:55:10.738 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern = core 2024-09-21T10:55:10.764 INFO:teuthology.orchestra.run.smithi045.stdout:kernel.core_pattern = core 2024-09-21T10:55:10.785 DEBUG:teuthology.orchestra.run.smithi032:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:55:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:10.811 DEBUG:teuthology.orchestra.run.smithi045:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:55:10.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:10.842 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T10:55:10.859 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T10:55:10.860 DEBUG:teuthology.misc:Transferring archived files from smithi032:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913660/remote/smithi032 2024-09-21T10:55:10.860 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:55:10.935 DEBUG:teuthology.misc:Transferring archived files from smithi045:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913660/remote/smithi045 2024-09-21T10:55:10.935 DEBUG:teuthology.orchestra.run.smithi045:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:55:10.981 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T10:55:10.981 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:55:10.983 DEBUG:teuthology.orchestra.run.smithi045:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:55:11.029 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T10:55:11.040 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T10:55:11.041 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T10:55:11.084 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T10:55:11.085 DEBUG:teuthology.orchestra.run.smithi032:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:55:11.087 DEBUG:teuthology.orchestra.run.smithi045:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:55:11.100 INFO:teuthology.orchestra.run.smithi032.stdout: 265324 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:55 /home/ubuntu/cephtest 2024-09-21T10:55:11.100 INFO:teuthology.orchestra.run.smithi032.stdout: 284955 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:33 /home/ubuntu/cephtest/mnt.0 2024-09-21T10:55:11.100 INFO:teuthology.orchestra.run.smithi032.stdout: 284956 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:33 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T10:55:11.100 INFO:teuthology.orchestra.run.smithi032.stdout: 424889 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:33 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:55:11.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:11.101 INFO:teuthology.orchestra.run.smithi032.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-21T10:55:11.102 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi032 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T10:55:11.108 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T10:55:11.167 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T10:55:11.227 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi032 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T10:55:11.231 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 4692.238032341003 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi032 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=1a404722ce8d4daeb2ea6b833e4a701d status: fail success: false 2024-09-21T10:55:11.232 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T10:55:11.239 INFO:teuthology.orchestra.run.smithi045.stdout: 265858 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:55 /home/ubuntu/cephtest 2024-09-21T10:55:11.329 INFO:teuthology.run:FAIL